def test_feed_ipv4_confidence():
x = Ipv4()
cli = Client('http://localhost:5000', 1234)
r = cli.aggregate(data, field='observable')
r = x.process(r, whitelist)
s = set()
for rr in r:
if rr['observable'] not in s:
s.add(rr['confidence'])
assert 75 not in s
def make_cifs_connection():
"""Connect to CIFS instance."""
cif_client = Client(token=psi_ops_config.CIF_TOKEN,
remote=psi_ops_config.CIF_REMOTE,
verify_ssl=psi_ops_config.CIF_VERIFY_SSL,
)
return cif_client
def main():
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif-apwg -v
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-apwg'
)
p.add_argument("-v", "--verbose", dest="verbose", action="count",
help="set verbosity level [default: %(default)s]")
p.add_argument('-d', '--debug', dest='debug', action="store_true")
p.add_argument("--config", dest="config", help="specify a configuration file [default: %(default)s]",
default=os.path.join(os.path.expanduser("~"), DEFAULT_CONFIG))
p.add_argument("--token", dest="token", help="specify token")
p.add_argument("--remote", dest="remote", help="specify the CIF remote")
p.add_argument("--group", dest="group", help="specify CIF group [default: %(default)s]", default="everyone")
p.add_argument('--no-verify-ssl', action="store_true", default=False)
# apwg options
p.add_argument("--limit", help="limit the number of records processed", default=LIMIT)
p.add_argument("--cache", default=os.path.join(os.path.expanduser("~"), ".cif/cleanmx"))
p.add_argument("--cleanmx-remote", default=CLEANMX_REMOTE)
p.add_argument('--tlp', default=TLP)
p.add_argument('--confidence', default=CONFIDENCE)
p.add_argument("--dry-run", help="do not submit to CIF", action="store_true")
p.add_argument("--no-last-run", help="do not modify lastrun file", action="store_true")
args = p.parse_args()
loglevel = logging.WARNING
if args.verbose:
loglevel = logging.INFO
if args.debug:
loglevel = logging.DEBUG
console = logging.StreamHandler()
logging.getLogger('').setLevel(loglevel)
console.setFormatter(logging.Formatter(LOG_FORMAT))
logging.getLogger('').addHandler(console)
logger = logging.getLogger(__name__)
options = vars(args)
if os.path.isfile(args.config):
f = file(args.config)
config = yaml.load(f)
f.close()
if not config['client']:
raise Exception("Unable to read " + args.config + " config file")
config = config['client']
for k in config:
if not options.get(k):
options[k] = config[k]
if not os.path.isdir(options["cache"]):
os.makedirs(options["cache"])
lastrun = os.path.join(options["cache"], "lastrun")
logger.debug(lastrun)
if os.path.exists(lastrun):
with open(lastrun) as f:
start = f.read().strip("\n")
uri = "{0}?delta={1}".format(
options["cleanmx_remote"],
start
)
else:
uri = "{0}?limit={1}".format(
options["cleanmx_remote"],
options['limit']
)
logger.debug("url: {0}".format(uri))
session = requests.Session()
session.headers['User-Agent'] = 'cif-cleanmx-py/0.0.0a1'
logger.info("pulling data")
try:
body = session.get(uri)
except requests.exceptions.ConnectionError as e:
logger.critical(e)
raise SystemExit
if len(body.content) > 2:
xmldoc = minidom.parseString(body.content)
items = xmldoc.getElementsByTagName('entry')
if len(items):
body = []
logger.debug('sending {0} items'.format(len(items)))
import arrow
reporttime = arrow.utcnow().format('YYYY-MM-DDTHH:mm:ss')
reporttime = '{}Z'.format(reporttime)
for e in reversed(items):
start = e.getElementsByTagName('id')[0].childNodes[0].data
body.append(
{
"observable": e.getElementsByTagName('url')[0].childNodes[0].data,
"lasttime": datetime.fromtimestamp(int(e.getElementsByTagName('first')[0].childNodes[0].data)).strftime(
"%Y-%m-%dT%H:%M:%SZ"),
"reporttime": reporttime,
"tags": ["phishing"],
"confidence": options["confidence"],
"tlp": options["tlp"],
"group": options["group"],
"otype": "url",
"provider": "support.clean-mx.de",
"application": ["http", "https"],
"altid": "http://support.clean-mx.de/clean-mx/phishing",
"altid_tlp": "green",
}
)
if not options.get("dry_run"):
logger.info("submitting {0} observables to CIF: {1}".format(len(body), options["remote"]))
cli = CIFClient(options['token'], remote=options['remote'], no_verify_ssl=options['no_verify_ssl'])
ret = cli.submit(json.dumps(body))
else:
pprint(body)
logger.info("dry run, skipping submit...")
else:
logger.info("nothing new to submit...")
if not options.get("no_last_run") and not options.get("dry_run"):
with open(os.path.join(options["cache"], "lastrun"), "w") as f:
f.write(str(start))
def test_client():
cli = Client(token=1234, remote='https://localhost2:8443', verify_ssl=False)
assert cli.verify_ssl is False
assert cli.remote == 'https://localhost2:8443','remote-mismatch'
assert cli.token == '1234', 'token mismatch'
def setUp(self):
self.cli = Client(token=1234,
remote='https://localhost2:8443',
no_verify_ssl=1)
def main():
#
# initialize module
#
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cat test.eml | cgmail
$ cgmail --file test.eml
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cgmail'
)
p.add_argument("-v", "--verbose", dest="verbose", action="count",
help="set verbosity level [default: %(default)s]")
p.add_argument('-d', '--debug', dest='debug', action="store_true")
p.add_argument("-f", "--file", dest="file", help="specify email file")
# cif arguments
p.add_argument("--confidence", help="specify confidence for submitting to CIF", default=CONFIDENCE)
p.add_argument("--remote", help="specify CIF remote [default: %(default)s",
default=REMOTE_DEFAULT)
p.add_argument("--token", help="specify CIF token")
p.add_argument("--config", help="specify CIF config [default: %(default)s",
default=os.path.expanduser("~/.cif.yml"))
p.add_argument("--tags", help="specify CIF tags [default: %(default)s", default=["phishing"])
p.add_argument("--group", help="specify CIF group [default: %(default)s", default="everyone")
p.add_argument("--tlp", help="specify CIF TLP [default: %(default)s", default=TLP)
p.add_argument("--no-verify-ssl", action="store_true", default=False)
p.add_argument("--raw", action="store_true", help="include raw message data")
p.add_argument("--provider", help="specify feed provider [default: %(default)s]", default=PROVIDER)
args = p.parse_args()
loglevel = logging.WARNING
if args.verbose:
loglevel = logging.INFO
if args.debug:
loglevel = logging.DEBUG
console = logging.StreamHandler()
logging.getLogger('').setLevel(loglevel)
console.setFormatter(logging.Formatter(LOG_FORMAT))
logging.getLogger('').addHandler(console)
logger = logging.getLogger(__name__)
options = vars(args)
if os.path.isfile(args.config):
f = file(args.config)
config = yaml.load(f)
f.close()
if not config['client']:
raise Exception("Unable to read " + args.config + " config file")
config = config['client']
for k in config:
if not options.get(k):
options[k] = config[k]
if config.get("remote") and options["remote"] == REMOTE_DEFAULT:
options["remote"] = config["remote"]
else:
logger.info("{} config does not exist".format(args.config))
#
# get email from file or stdin
#
if not options.get("remote"):
logger.critical("missing --remote")
raise SystemExit
if options.get("file"):
with open(options["file"]) as f:
email = f.read()
else:
email = sys.stdin.read()
# extract urls from message body and mail parts
urls = cgmail.extract_urls(email)
#
# submit urls to a CIF instance
#
# initialize cif client
cli = Client(remote=options["remote"], token=options["token"], no_verify_ssl=options["no_verify_ssl"])
for u in urls:
logger.info("submitting: {0}".format(u))
o = Observable(
observable=u,
confidence=options["confidence"],
tlp=options["tlp"],
group=options["group"],
tags=options["tags"],
provider=options.get('provider')
)
if options.get('raw'):
o.raw = email
r = cli.submit(str(o))
logger.info("submitted: {0}".format(r))
def main():
#
# initialize module
#
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cat test.eml | cgmail
$ cgmail --file test.eml
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cgmail'
)
p.add_argument("-v", "--verbose", dest="verbose", action="count",
help="set verbosity level [default: %(default)s]")
p.add_argument('-d', '--debug', dest='debug', action="store_true")
p.add_argument("-f", "--file", dest="file", help="specify email file")
# cif arguments
p.add_argument("--confidence", help="specify confidence for submitting to CIF", default=CONFIDENCE)
p.add_argument("--remote", help="specify CIF remote")
p.add_argument("--token", help="specify CIF token")
p.add_argument("--config", help="specify CIF config [default: %(default)s",
default=os.path.expanduser("~/.cif.yml"))
p.add_argument("--tags", help="specify CIF tags [default: %(default)s", default=["phishing"])
p.add_argument("--group", help="specify CIF group [default: %(default)s", default="everyone")
p.add_argument("--tlp", help="specify CIF TLP [default: %(default)s", default=TLP)
p.add_argument("--no-verify-ssl", action="store_true", default=False)
p.add_argument("--raw", action="store_true", help="include raw message data")
p.add_argument("--provider", help="specify feed provider [default: %(default)s]", default=PROVIDER)
p.add_argument('--exclude', help='url patterns to exclude [default: %(default)s', default=EXCLUDE)
p.add_argument('--confidence-lower', help='patterns to automatically lower confidence', default=CONFIDENCE_LOWER)
p.add_argument('-n', '--not-really', help='do not submit', action='store_true')
p.add_argument('--cache', help='location to cache whitelist [default: %(default)s', default=WHITELIST_CACHE)
p.add_argument('--blacklist-cache', default=BLACKLIST_CACHE)
# Process arguments
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
exclude = None
if args.exclude:
exclude = re.compile(args.exclude)
confidence_lower = None
if args.confidence_lower:
confidence_lower = re.compile(args.confidence_lower)
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
if not options.get('token'):
raise RuntimeError('missing --token')
if options.get("file"):
with open(options["file"]) as f:
email = f.read()
else:
email = sys.stdin.read()
# extract urls from message body and mail parts
bits = cgmail.parse_email_from_string(email)
urls = set()
for n in bits:
if n.get('urls'):
for u in n['urls']:
urls.add(u)
verify_ssl = True
if options.get('no_verify_ssl'):
verify_ssl = False
# initialize cif client
cli = Client(remote=options["remote"], token=options["token"], verify_ssl=verify_ssl)
update_cache = True
if os.path.isfile(args.cache):
modified = os.path.getmtime(args.cache)
if arrow.utcnow() < arrow.get(modified + 84600):
update_cache = False
if update_cache:
# pull FQDN whitelist
filters = {
'tags': 'whitelist',
'otype': 'fqdn',
'confidence': 25,
}
now = arrow.utcnow()
filters['reporttimeend'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.replace(days=-7)
filters['reporttime'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
ret = cli.search(limit=50000, filters=filters, sort='reporttime', sort_direction='desc')
with open(args.cache, 'w') as f:
for r in ret:
f.write("{0}\n".format(r['observable']))
update_cache = True
if os.path.isfile(args.blacklist_cache):
modified = os.path.getmtime(args.blacklist_cache)
if arrow.utcnow() < arrow.get(modified + 84600):
update_cache = False
if update_cache:
filters = {
'tags': 'phishing,suspicious,malware',
'otype': 'fqdn',
'confidence': 75,
}
now = arrow.utcnow()
filters['reporttimeend'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.replace(days=-7)
filters['reporttime'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
ret = cli.search(limit=50000, filters=filters, sort='reporttime', sort_direction='desc')
with open(args.blacklist_cache, 'w') as f:
for r in ret:
f.write("{0}\n".format(r['observable']))
fqdns = set()
with open(args.cache) as f:
for l in f:
fqdns.add(l.rstrip("\n"))
fqdns_blacklist = set()
with open(args.blacklist_cache) as f:
for l in f:
fqdns_blacklist.add(l.rstrip("\n"))
for u in urls:
u = u.rstrip('\/')
u = urlparse(u)
fqdn = url_to_fqdn(u.geturl())
if exclude and exclude.search(fqdn):
continue
confidence = options['confidence']
if match_whitelist(fqdns, u.netloc):
if (u.netloc not in URL_SHORTNERS) and (not match_whitelist(HOSTING_PROVIDERS, u.netloc)):
confidence = options['confidence'] - 15
else:
confidence = options['confidence'] + 5
elif match_whitelist(fqdns_blacklist, u.netloc):
confidence = options['confidence'] + 10
else:
confidence = options['confidence'] + 5
# else
# raise confidence
logger.info("submitting: {0}".format(u.geturl()))
o = Observable(
observable=u.geturl(),
confidence=confidence,
tlp=options["tlp"],
group=options["group"],
tags=options["tags"],
provider=options.get('provider')
)
o = o.__dict__
del o['logger']
if options.get('raw'):
o.raw = email
if not args.not_really:
r = cli.submit(o)
logger.info("submitted: {0}".format(r))
def main():
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ export APWG_REMOTE=https://api.apwg.org/endpoint
$ export APWG_TOKEN=123123123
$ export CIF_REMOTE=https://csirtgadgets.org
$ export CIF_TOKEN=123412341234
$ cif-apwg-submit -v
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-apwg-submit'
)
p.add_argument("-v", "--verbose", dest="verbose", action="count",
help="set verbosity level [default: %(default)s]")
p.add_argument('-d', '--debug', dest='debug', action="store_true")
p.add_argument('--no-verify-ssl', action="store_true", default=False)
# apwg options
p.add_argument("--limit", dest="limit", help="limit the number of records processed")
p.add_argument("--cache", default=os.path.join(os.path.expanduser("~"), ".cif/apwg_submit"))
p.add_argument("--past-minutes", help="number of hours to go back and retrieve", default=10)
p.add_argument("--apwg-confidence", default="90")
# cif options
p.add_argument('--tlp', default=TLP)
p.add_argument('--altid-tlp', default=TLP)
p.add_argument('--altid')
p.add_argument('--confidence', default=CONFIDENCE)
p.add_argument('--tags', default="phishing")
p.add_argument('--otype', default='url')
p.add_argument('--itype', default='url')
p.add_argument('--filters', default=None)
p.add_argument('--group', default="everyone")
p.add_argument("--dry-run", help="do not submit to CIF", action="store_true")
p.add_argument("--no-last-run", help="do not modify lastrun file", action="store_true")
args = p.parse_args()
filters = {
"confidence": args.confidence,
"tags": args.tags.split(','),
"otype": args.otype,
"itype": args.itype,
"group": args.group,
}
if args.filters:
for e in args.filters.split('&', 1):
k = e.split('=', 1)
filters[k[0]] = k[1]
loglevel = logging.WARNING
if args.verbose:
loglevel = logging.INFO
if args.debug:
loglevel = logging.DEBUG
console = logging.StreamHandler()
logging.getLogger('').setLevel(loglevel)
console.setFormatter(logging.Formatter(LOG_FORMAT))
logging.getLogger('').addHandler(console)
logger = logging.getLogger(__name__)
if not APWG_REMOTE and not args.dry_run:
print("\nAPWG_REMOTE missing.\nConsult with [email protected] for more information")
raise SystemExit
if not os.path.isdir(args.cache):
os.makedirs(args.cache)
end = datetime.utcnow()
lastrun = os.path.join(args.cache, "lastrun")
logger.debug(lastrun)
if os.path.exists(lastrun):
with open(lastrun) as f:
start = f.read().strip("\n")
start = datetime.strptime(start, '%Y-%m-%d %H:%M:%S.%f')
else:
minutes = int(args.past_minutes)
start = end - timedelta(minutes=minutes, seconds=-1)
start = start.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
end = end.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
logger.info("start:{0}".format(start))
logger.info("end:{0}".format(end))
filters['reporttime'] = start
filters['reporttimeend'] = end
logger.debug(json.dumps(filters, indent=4))
# get data
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
from apwgsdk.client import Client as APWGClient
apwg = APWGClient()
cli = Client(CIF_TOKEN, remote=CIF_REMOTE, verify_ssl=verify_ssl)
data = cli.search(limit=args.limit, filters=filters)
for d in data:
if d.get('observable'):
d['indicator'] = d['observable']
if args.altid and not d.get('altid', '').startswith(args.altid):
logger.debug('skipping.. %s' % d['indicator'])
# if the altid doesn't match, skip it..
continue
if logger.getEffectiveLevel() == logging.DEBUG:
logger.debug("%s" % json.dumps(d, indent=4))
if args.dry_run:
logger.info(json.dumps(d, indent=4))
continue
if logger.getEffectiveLevel() == logging.DEBUG:
logger.debug('sending: %s' % json.dumps(d, indent=4))
else:
logger.info('sending: %s' % d['indicator'])
# submit to apwg
try:
r = apwg.indicators_create(indicator=d['indicator'], confidence=args.apwg_confidence,
description='phishing', lasttime=d['lasttime'])
logger.info('indicator created successfully: {}'.format(r['id']))
if args.debug:
pprint(r)
except Exception as e:
logger.debug(e)
logger.error('error creating indicator')
if not args.no_last_run and not args.dry_run:
with open(os.path.join(args.cache, "lastrun"), "w") as f:
f.write(str(end))
def main():
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ export APWG_REMOTE=https://api.apwg.org/endpoint
$ export APWG_TOKEN=123123123
$ cif-apwg -v
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-apwg'
)
p.add_argument("-v", "--verbose", dest="verbose", action="count",
help="set verbosity level [default: %(default)s]")
p.add_argument('-d', '--debug', dest='debug', action="store_true")
p.add_argument("--config", dest="config", help="specify a configuration file [default: %(default)s]",
default=os.path.join(os.path.expanduser("~"), DEFAULT_CONFIG))
p.add_argument("--token", dest="token", help="specify token")
p.add_argument("--remote", dest="remote", help="specify the CIF remote")
p.add_argument("--group", dest="group", help="specify CIF group [default: %(default)s]", default="everyone")
p.add_argument('--no-verify-ssl', action="store_true", default=False)
# apwg options
p.add_argument("--limit", dest="limit", help="limit the number of records processed")
p.add_argument("--apwg-token", help="specify an APWG token", required=True, default=APWG_TOKEN)
p.add_argument("--format", default="json")
p.add_argument("--cache", default=os.path.join(os.path.expanduser("~"), ".cif/apwg"))
p.add_argument("--apwg-remote", default=APWG_REMOTE, required=True)
p.add_argument("--past-hours", help="number of hours to go back and retrieve", default="24")
p.add_argument("--apwg-confidence-low", default="65")
p.add_argument("--apwg-confidence-high", default="100")
p.add_argument('--tlp', default=TLP)
p.add_argument('--altid-tlp', default=TLP)
p.add_argument('--confidence', default=CONFIDENCE)
p.add_argument("--dry-run", help="do not submit to CIF", action="store_true")
p.add_argument("--no-last-run", help="do not modify lastrun file", action="store_true")
args = p.parse_args()
loglevel = logging.WARNING
if args.verbose:
loglevel = logging.INFO
if args.debug:
loglevel = logging.DEBUG
console = logging.StreamHandler()
logging.getLogger('').setLevel(loglevel)
console.setFormatter(logging.Formatter(LOG_FORMAT))
logging.getLogger('').addHandler(console)
logger = logging.getLogger(__name__)
if not args.apwg_remote:
print("\nAPWG_REMOTE missing.\nConsult with [email protected] for more information")
raise SystemExit
options = vars(args)
if os.path.isfile(args.config):
f = file(args.config)
config = yaml.load(f)
f.close()
if not config['client']:
raise Exception("Unable to read " + args.config + " config file")
config = config['client']
for k in config:
if not options.get(k):
options[k] = config[k]
if not os.path.isdir(options["cache"]):
os.makedirs(options["cache"])
end = datetime.utcnow()
lastrun = os.path.join(options["cache"], "lastrun")
logger.debug(lastrun)
if os.path.exists(lastrun):
with open(lastrun) as f:
start = f.read().strip("\n")
start = datetime.strptime(start, '%Y-%m-%d %H:%M:%S.%f')
else:
hours = int(options["past_hours"])
start = end - timedelta(hours=hours, seconds=-1)
logger.info("start:{0}".format(start))
logger.info("end:{0}".format(end))
uri = "{}?t={}&dd_date_start={}&dd_date_end={}&confidence_low=90".format(
options['apwg_remote'],
options["apwg_token"],
start.strftime('%s'),
end.strftime('%s')
)
logger.debug("apwg url: {0}".format(uri))
session = requests.Session()
session.headers['User-Agent'] = 'cif-apwg-py/0.0.3a'
logger.info("pulling apwg data")
body = session.get(uri)
body = json.loads(body.content)
body = body['_embedded']['phish']
if len(body):
if options.get("limit"):
body = body[-int(options["limit"]):]
body = [
{
"observable": e["url"].lower().lstrip('3d'),
"reporttime": datetime.fromtimestamp(e["modified"]).strftime("%Y-%m-%dT%H:%M:%SZ"),
"firsttime": datetime.fromtimestamp(e["date_discovered"]).strftime("%Y-%m-%dT%H:%M:%SZ"),
"lasttime": datetime.fromtimestamp(e['date_discovered']).strftime("%Y-%m-%dT%H:%M:%SZ"),
"tags": ["phishing"],
"description": e["brand"].lower(),
"confidence": options["confidence"],
"tlp": options["tlp"],
"group": options["group"],
"otype": "url",
"provider": "apwg.org",
"altid_tlp": options['altid_tlp'],
"application": ["http", "https"]
} for e in reversed(body)]
logger.info("start of data: {0}".format(body[len(body)-1]["reporttime"]))
logger.info("end of data: {0}".format(body[0]["reporttime"]))
if not options.get("dry_run"):
logger.info("submitting {0} observables to CIF: {1}".format(len(body), options["remote"]))
verify_ssl = True
if options['no_verify_ssl']:
verify_ssl = False
cli = Client(options['token'], remote=options['remote'], verify_ssl=verify_ssl)
ret = cli.submit(body)
else:
pprint(body)
logger.info("dry run, skipping submit...{}".format(len(body)))
else:
logger.info("nothing new to submit...")
if not options.get("no_last_run") and not options.get("dry_run"):
with open(os.path.join(options["cache"], "lastrun"), "w") as f:
f.write(str(end))
def main():
#
# initialize module
#
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cat test.eml | cgmail
$ cgmail --file test.eml
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cgmail')
p.add_argument("-v",
"--verbose",
dest="verbose",
action="count",
help="set verbosity level [default: %(default)s]")
p.add_argument('-d', '--debug', dest='debug', action="store_true")
p.add_argument("-f", "--file", dest="file", help="specify email file")
# cif arguments
p.add_argument("--confidence",
help="specify confidence for submitting to CIF",
default=CONFIDENCE)
p.add_argument("--remote", help="specify CIF remote")
p.add_argument("--token", help="specify CIF token")
p.add_argument("--config",
help="specify CIF config [default: %(default)s",
default=os.path.expanduser("~/.cif.yml"))
p.add_argument("--tags",
help="specify CIF tags [default: %(default)s",
default=["phishing"])
p.add_argument("--group",
help="specify CIF group [default: %(default)s",
default="everyone")
p.add_argument("--tlp",
help="specify CIF TLP [default: %(default)s",
default=TLP)
p.add_argument("--no-verify-ssl", action="store_true", default=False)
p.add_argument("--raw",
action="store_true",
help="include raw message data")
p.add_argument("--provider",
help="specify feed provider [default: %(default)s]",
default=PROVIDER)
p.add_argument('--exclude',
help='url patterns to exclude [default: %(default)s',
default=EXCLUDE)
p.add_argument('--confidence-lower',
help='patterns to automatically lower confidence',
default=CONFIDENCE_LOWER)
p.add_argument('-n',
'--not-really',
help='do not submit',
action='store_true')
p.add_argument('--cache',
help='location to cache whitelist [default: %(default)s',
default=WHITELIST_CACHE)
p.add_argument('--blacklist-cache', default=BLACKLIST_CACHE)
# Process arguments
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
exclude = None
if args.exclude:
exclude = re.compile(args.exclude)
confidence_lower = None
if args.confidence_lower:
confidence_lower = re.compile(args.confidence_lower)
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
if not options.get('token'):
raise RuntimeError('missing --token')
if options.get("file"):
with open(options["file"]) as f:
email = f.read()
else:
email = sys.stdin.read()
# extract urls from message body and mail parts
bits = cgmail.parse_email_from_string(email)
urls = set()
for n in bits:
if n.get('urls'):
for u in n['urls']:
urls.add(u)
verify_ssl = True
if options.get('no_verify_ssl'):
verify_ssl = False
# initialize cif client
cli = Client(remote=options["remote"],
token=options["token"],
verify_ssl=verify_ssl)
update_cache = True
if os.path.isfile(args.cache):
modified = os.path.getmtime(args.cache)
if arrow.utcnow() < arrow.get(modified + 84600):
update_cache = False
if update_cache:
# pull FQDN whitelist
filters = {
'tags': 'whitelist',
'otype': 'fqdn',
'confidence': 25,
}
now = arrow.utcnow()
filters['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.shift(days=-7)
filters['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
ret = cli.search(limit=50000,
filters=filters,
sort='reporttime',
sort_direction='desc')
with open(args.cache, 'w') as f:
for r in ret:
f.write("{0}\n".format(r['observable']))
update_cache = True
if os.path.isfile(args.blacklist_cache):
modified = os.path.getmtime(args.blacklist_cache)
if arrow.utcnow() < arrow.get(modified + 84600):
update_cache = False
if update_cache:
filters = {
'tags': 'phishing,suspicious,malware',
'otype': 'fqdn',
'confidence': 75,
}
now = arrow.utcnow()
filters['reporttimeend'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
now = now.shift(days=-7)
filters['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDTHH:mm:ss'))
ret = cli.search(limit=50000,
filters=filters,
sort='reporttime',
sort_direction='desc')
with open(args.blacklist_cache, 'w') as f:
for r in ret:
f.write("{0}\n".format(r['observable']))
fqdns = set()
with open(args.cache) as f:
for l in f:
fqdns.add(l.rstrip("\n"))
fqdns_blacklist = set()
with open(args.blacklist_cache) as f:
for l in f:
fqdns_blacklist.add(l.rstrip("\n"))
for u in urls:
u = u.rstrip('\/')
u = urlparse(u)
fqdn = url_to_fqdn(u.geturl())
if exclude and exclude.search(fqdn):
continue
confidence = options['confidence']
if match_whitelist(fqdns, u.netloc):
if (u.netloc not in URL_SHORTNERS) and (not match_whitelist(
HOSTING_PROVIDERS, u.netloc)):
confidence = options['confidence'] - 15
else:
confidence = options['confidence'] + 5
elif match_whitelist(fqdns_blacklist, u.netloc):
confidence = options['confidence'] + 10
else:
confidence = options['confidence'] + 5
# else
# raise confidence
logger.info("submitting: {0}".format(u.geturl()))
o = Observable(observable=u.geturl(),
confidence=confidence,
tlp=options["tlp"],
group=options["group"],
tags=options["tags"],
provider=options.get('provider'))
o = o.__dict__
del o['logger']
if options.get('raw'):
o.raw = email
if not args.not_really:
r = cli.submit(o)
logger.info("submitted: {0}".format(r))
def main():
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ export APWG_REMOTE=https://api.apwg.org/endpoint
$ export APWG_TOKEN=123123123
$ export CIF_REMOTE=https://csirtgadgets.org
$ export CIF_TOKEN=123412341234
$ cif-apwg-submit -v
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-apwg-submit')
p.add_argument("-v",
"--verbose",
dest="verbose",
action="count",
help="set verbosity level [default: %(default)s]")
p.add_argument('-d', '--debug', dest='debug', action="store_true")
p.add_argument('--no-verify-ssl', action="store_true", default=False)
# apwg options
p.add_argument("--limit",
dest="limit",
help="limit the number of records processed")
p.add_argument("--cache",
default=os.path.join(os.path.expanduser("~"),
".cif/apwg_submit"))
p.add_argument("--past-minutes",
help="number of hours to go back and retrieve",
default=10)
p.add_argument("--apwg-confidence", default="90")
# cif options
p.add_argument('--tlp', default=TLP)
p.add_argument('--altid-tlp', default=TLP)
p.add_argument('--altid')
p.add_argument('--confidence', default=CONFIDENCE)
p.add_argument('--tags', default="phishing")
p.add_argument('--otype', default='url')
p.add_argument('--itype', default='url')
p.add_argument('--filters', default=None)
p.add_argument('--group', default="everyone")
p.add_argument("--dry-run",
help="do not submit to CIF",
action="store_true")
p.add_argument("--no-last-run",
help="do not modify lastrun file",
action="store_true")
args = p.parse_args()
filters = {
"confidence": args.confidence,
"tags": args.tags.split(','),
"otype": args.otype,
"itype": args.itype,
"group": args.group,
}
if args.filters:
for e in args.filters.split('&', 1):
k = e.split('=', 1)
filters[k[0]] = k[1]
loglevel = logging.WARNING
if args.verbose:
loglevel = logging.INFO
if args.debug:
loglevel = logging.DEBUG
console = logging.StreamHandler()
logging.getLogger('').setLevel(loglevel)
console.setFormatter(logging.Formatter(LOG_FORMAT))
logging.getLogger('').addHandler(console)
logger = logging.getLogger(__name__)
if not APWG_REMOTE and not args.dry_run:
print(
"\nAPWG_REMOTE missing.\nConsult with [email protected] for more information"
)
raise SystemExit
if not os.path.isdir(args.cache):
os.makedirs(args.cache)
end = datetime.utcnow()
lastrun = os.path.join(args.cache, "lastrun")
logger.debug(lastrun)
if os.path.exists(lastrun):
with open(lastrun) as f:
start = f.read().strip("\n")
start = datetime.strptime(start, '%Y-%m-%d %H:%M:%S.%f')
else:
minutes = int(args.past_minutes)
start = end - timedelta(minutes=minutes, seconds=-1)
start = start.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
end = end.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
logger.info("start:{0}".format(start))
logger.info("end:{0}".format(end))
filters['reporttime'] = start
filters['reporttimeend'] = end
logger.debug(json.dumps(filters, indent=4))
# get data
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
from apwgsdk.client import Client as APWGClient
apwg = APWGClient()
cli = Client(CIF_TOKEN, remote=CIF_REMOTE, verify_ssl=verify_ssl)
data = cli.search(limit=args.limit, filters=filters)
for d in data:
if d.get('observable'):
d['indicator'] = d['observable']
if args.altid and not d.get('altid', '').startswith(args.altid):
logger.debug('skipping.. %s' % d['indicator'])
# if the altid doesn't match, skip it..
continue
if logger.getEffectiveLevel() == logging.DEBUG:
logger.debug("%s" % json.dumps(d, indent=4))
if args.dry_run:
logger.info(json.dumps(d, indent=4))
continue
if logger.getEffectiveLevel() == logging.DEBUG:
logger.debug('sending: %s' % json.dumps(d, indent=4))
else:
logger.info('sending: %s' % d['indicator'])
# submit to apwg
try:
r = apwg.indicators_create(indicator=d['indicator'],
confidence=args.apwg_confidence,
description='phishing',
lasttime=d['lasttime'])
logger.info('indicator created successfully: {}'.format(r['id']))
if args.debug:
pprint(r)
except Exception as e:
logger.debug(e)
logger.error('error creating indicator')
if not args.no_last_run and not args.dry_run:
with open(os.path.join(args.cache, "lastrun"), "w") as f:
f.write(str(end))
def main():
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cat test.eml | cif-procmail -v
$ cif-procmail --file test.eml
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-procmail'
)
p.add_argument("-v", "--verbose", dest="verbose", action="count",
help="set verbosity level [default: %(default)s]")
p.add_argument('-d', '--debug', dest='debug', action="store_true")
p.add_argument("-f", "--file", dest="file", help="specify email file")
# cif arguments
p.add_argument("--confidence", help="specify confidence for submitting to CIF", default="65")
p.add_argument("--remote", help="specify CIF remote [default: %(default)s",
default=REMOTE_DEFAULT)
p.add_argument("--token", help="specify CIF token")
p.add_argument("--config", help="specify CIF config [default: %(default)s",
default=os.path.expanduser("~/.cif.yml"))
p.add_argument("--tags", help="specify CIF tags [default: %(default)s", default=["phishing"])
p.add_argument("--group", help="specify CIF group [default: %(default)s", default="everyone")
p.add_argument("--tlp", help="specify CIF TLP [default: %(default)s", default="amber")
p.add_argument("--no-verify-ssl", action="store_true", default=False)
p.add_argument("--raw", help="include raw message data")
p.add_argument("--raw-headers", help="include raw header data", action="store_true")
p.add_argument("--provider", help="specify feed provider [default: %(default)s", default="localhost")
args = p.parse_args()
loglevel = logging.WARNING
if args.verbose:
loglevel = logging.INFO
if args.debug:
loglevel = logging.DEBUG
console = logging.StreamHandler()
logging.getLogger('').setLevel(loglevel)
console.setFormatter(logging.Formatter(LOG_FORMAT))
logging.getLogger('').addHandler(console)
logger = logging.getLogger(__name__)
options = vars(args)
if os.path.isfile(args.config):
f = file(args.config)
config = yaml.load(f)
f.close()
if not config['client']:
raise Exception("Unable to read " + args.config + " config file")
config = config['client']
for k in config:
if not options.get(k):
options[k] = config[k]
if config.get("remote") and options["remote"] == REMOTE_DEFAULT:
options["remote"] = config["remote"]
else:
logger.info("{} config does not exist".format(args.config))
if not options.get("remote"):
logger.critical("missing --remote")
raise SystemExit
if options.get("file"):
with open(options["file"]) as f:
msg = email.message_from_file(f)
else:
msg = sys.stdin.read()
msg = email.message_from_string(msg)
urls = set()
# plain text vs html
#print msg.get_content_type()
if msg.is_multipart():
msgs = msg.get_payload()
for i, m in enumerate(msgs[1:]):
html = str(m.get_payload()) ## TODO
urls.update(extract_urls(html))
else:
content = msg.get_payload()
if not msg.get_content_type.startswith("text/plain"):
html = True
urls.update(extract_urls(content, html=html))
pprint(urls)
cli = Client(remote=options["remote"], token=options["token"], no_verify_ssl=options["no_verify_ssl"])
for u in urls:
logger.info("submitting: {0}".format(u))
raw = html
if options.get("raw_headers"):
raw = msg.as_string()
o = Observable(
observable=u,
confidence=options["confidence"],
tlp=options["tlp"],
group=options["group"],
tags=options["tags"],
raw=raw,
provider=options["provider"]
)
r = cli.submit(submit=str(o))
logger.info("submitted: {0}".format(r))
def main():
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ export APWG_REMOTE=https://api.apwg.org/endpoint
$ export APWG_TOKEN=123123123
$ cif-apwg -v
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-apwg')
p.add_argument("-v",
"--verbose",
dest="verbose",
action="count",
help="set verbosity level [default: %(default)s]")
p.add_argument('-d', '--debug', dest='debug', action="store_true")
p.add_argument("--config",
dest="config",
help="specify a configuration file [default: %(default)s]",
default=os.path.join(os.path.expanduser("~"),
DEFAULT_CONFIG))
p.add_argument("--token", dest="token", help="specify token")
p.add_argument("--remote", dest="remote", help="specify the CIF remote")
p.add_argument("--group",
dest="group",
help="specify CIF group [default: %(default)s]",
default="everyone")
p.add_argument('--no-verify-ssl', action="store_true", default=False)
# apwg options
p.add_argument("--limit",
dest="limit",
help="limit the number of records processed")
p.add_argument("--apwg-token",
help="specify an APWG token",
required=True,
default=APWG_TOKEN)
p.add_argument("--format", default="json")
p.add_argument("--cache",
default=os.path.join(os.path.expanduser("~"), ".cif/apwg"))
p.add_argument("--apwg-remote", default=APWG_REMOTE, required=True)
p.add_argument("--past-hours",
help="number of hours to go back and retrieve",
default="24")
p.add_argument("--apwg-confidence-low", default="65")
p.add_argument("--apwg-confidence-high", default="100")
p.add_argument('--tlp', default=TLP)
p.add_argument('--altid-tlp', default=TLP)
p.add_argument('--confidence', default=CONFIDENCE)
p.add_argument("--dry-run",
help="do not submit to CIF",
action="store_true")
p.add_argument("--no-last-run",
help="do not modify lastrun file",
action="store_true")
args = p.parse_args()
loglevel = logging.WARNING
if args.verbose:
loglevel = logging.INFO
if args.debug:
loglevel = logging.DEBUG
console = logging.StreamHandler()
logging.getLogger('').setLevel(loglevel)
console.setFormatter(logging.Formatter(LOG_FORMAT))
logging.getLogger('').addHandler(console)
logger = logging.getLogger(__name__)
if not args.apwg_remote:
print(
"\nAPWG_REMOTE missing.\nConsult with [email protected] for more information"
)
raise SystemExit
options = vars(args)
if os.path.isfile(args.config):
f = file(args.config)
config = yaml.load(f)
f.close()
if not config['client']:
raise Exception("Unable to read " + args.config + " config file")
config = config['client']
for k in config:
if not options.get(k):
options[k] = config[k]
if not os.path.isdir(options["cache"]):
os.makedirs(options["cache"])
end = datetime.utcnow()
lastrun = os.path.join(options["cache"], "lastrun")
logger.debug(lastrun)
if os.path.exists(lastrun):
with open(lastrun) as f:
start = f.read().strip("\n")
start = datetime.strptime(start, '%Y-%m-%d %H:%M:%S.%f')
else:
hours = int(options["past_hours"])
start = end - timedelta(hours=hours, seconds=-1)
logger.info("start:{0}".format(start))
logger.info("end:{0}".format(end))
uri = "{}?t={}&dd_date_start={}&dd_date_end={}&confidence_low=90".format(
options['apwg_remote'], options["apwg_token"], start.strftime('%s'),
end.strftime('%s'))
logger.debug("apwg url: {0}".format(uri))
session = requests.Session()
session.headers['User-Agent'] = 'cif-apwg-py/0.0.3a'
logger.info("pulling apwg data")
body = session.get(uri)
body = json.loads(body.content)
body = body['_embedded']['phish']
if len(body):
if options.get("limit"):
body = body[-int(options["limit"]):]
body = [{
"observable":
e["url"].lower().lstrip('3d'),
"reporttime":
datetime.fromtimestamp(
e["modified"]).strftime("%Y-%m-%dT%H:%M:%SZ"),
"firsttime":
datetime.fromtimestamp(
e["date_discovered"]).strftime("%Y-%m-%dT%H:%M:%SZ"),
"lasttime":
datetime.fromtimestamp(
e['date_discovered']).strftime("%Y-%m-%dT%H:%M:%SZ"),
"tags": ["phishing"],
"description":
e["brand"].lower(),
"confidence":
options["confidence"],
"tlp":
options["tlp"],
"group":
options["group"],
"otype":
"url",
"provider":
"apwg.org",
"altid_tlp":
options['altid_tlp'],
"application": ["http", "https"]
} for e in reversed(body)]
logger.info("start of data: {0}".format(body[len(body) -
1]["reporttime"]))
logger.info("end of data: {0}".format(body[0]["reporttime"]))
if not options.get("dry_run"):
logger.info("submitting {0} observables to CIF: {1}".format(
len(body), options["remote"]))
verify_ssl = True
if options['no_verify_ssl']:
verify_ssl = False
cli = Client(options['token'],
remote=options['remote'],
verify_ssl=verify_ssl)
ret = cli.submit(body)
else:
pprint(body)
logger.info("dry run, skipping submit...{}".format(len(body)))
else:
logger.info("nothing new to submit...")
if not options.get("no_last_run") and not options.get("dry_run"):
with open(os.path.join(options["cache"], "lastrun"), "w") as f:
f.write(str(end))
