def parseargs() -> argparse.ArgumentParser:
""" Parse arguments """
parser = worker.parseargs('Get MISP feeds from MISP sharing directories')
parser.add_argument('--manifest-dir', default=worker.get_cache_dir('misp_manifest'),
help='The directory to store latest manifests')
return parser
def update_last_update(last_update: int) -> None:
"Write last update from disk (~/.cache/<worker_name>/last_update)"
cache_filename: Text = os.path.join(
worker.get_cache_dir(worker.worker_name(), create=True), "last_update")
# Write last update timestamp to disk
with open(cache_filename, "w") as f:
f.write(str(last_update))
def verify_manifest_dir(manifest_dir: Text) -> None:
"""Verify that the directory structure exists and that there is
always a feed file (Even empty)"""
# Manifest is at default location - create directory if it does not exists
if manifest_dir == worker.get_cache_dir('misp_manifest'):
worker.get_cache_dir('misp_manifest', create=True)
# If there is specified a manifest directory in the .ini file we
# verify that it exists (or fail hard). If no such directory
# is defined, we default to using $XDG_CACHE_DIR and create a new
# 'misp_maifest' directory there.
if not os.path.isdir(manifest_dir):
print("Could not open manifest directory:", manifest_dir)
sys.exit(1)
# Check that the misp_feeds.txt file actually exists. If not 'touch'
# the file to make sure there is at least some default config present.
feed_file = os.path.join(manifest_dir, 'misp_feeds.txt')
if not os.path.isfile(feed_file):
with open(feed_file, 'w') as feed_h:
feed_h.write("https://www.circl.lu/doc/misp/feed-osint/")
def get_last_update() -> int:
"Get last update from disk (~/.cache/<worker_name>/last_update)"
cache_filename: Text = os.path.join(
worker.get_cache_dir(worker.worker_name(), create=True), "last_update")
if os.path.isfile(cache_filename):
# Read last_update from last recorded succsfully recieved event
with open(cache_filename) as f:
last_update = int(f.read().strip())
debug("last update starting at {}".format(last_update))
else:
# last_update not specified, set to now-1w
last_update = int((time.time() - 3600 * 24 * 7) * 1000)
info("last update not specified, autoconfigured as {}".format(
last_update))
return last_update
import socket
import sqlite3
import sys
import time
import traceback
from ipaddress import AddressValueError, IPv4Address
from logging import debug, error, info, warning
from typing import Dict, Generator, List, Text, Tuple, Union
from RashlyOutlaid.libwhois import ASNRecord, ASNWhois, QueryError
import act.api
from act.api.helpers import handle_fact
from act.workers.libs import worker
CACHE_DIR = worker.get_cache_dir("shadowserver-asn-worker", create=True)
VERSION = "0.1"
ISO_3166_FILE = "https://raw.githubusercontent.com/lukes/" + \
"ISO-3166-Countries-with-Regional-Codes/master/all/all.json"
# Blacklists of IPs record values
# If value matches blacklist it should not be used
BLACKLIST = {
"ip": [ # Blacklist IP addresses. Values is IP
lambda ip: not ip.strip(), # Empty values
lambda ip: ip.strip().lstrip("0").startswith(
"."), # IP addreses starting with "0."
lambda ip: ip == "255.255.255.255", # broadcast
lambda ip: IPv4Address(ip).is_multicast,
lambda ip: IPv4Address(ip).is_private,
lambda ip: IPv4Address(ip).is_loopback,