def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH):
create_directory(FILE_OUTPUT_PATH, 0o40755)
create_directory(VAR_RUN_PATH, 0o40755)
create_directory(DEFAULT_LOGGING_PATH, 0o40755)
create_directory(CONFIG_DIR, 0o40775)
example_path = Path(pkg_resources.resource_filename('intelmq', 'etc'))
example_confs = [example_path / 'runtime.yaml', example_path / 'harmonization.conf']
for example_conf in example_confs:
fname = Path(example_conf).name
destination_file = Path(CONFIG_DIR) / fname
if destination_file.exists():
print(f'Not overwriting existing {fname!r} with example.')
log_ownership_change = True
else:
shutil.copy(example_conf, CONFIG_DIR)
print(f'Installing example {fname!r} to {CONFIG_DIR}.')
log_ownership_change = False # For installing the new files, we don't need to inform the admin that the permissions have been "fixed"
if ownership:
change_owner(destination_file, owner='intelmq', group='intelmq', log=log_ownership_change)
if ownership:
print('Setting intelmq as owner for it\'s directories.')
for obj in (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
VAR_STATE_PATH, FILE_OUTPUT_PATH, Path(STATE_FILE_PATH).parent):
change_owner(obj, owner='intelmq')
print('Calling `intelmqctl upgrade-config` to update/create state file.')
controller = IntelMQController(interactive=False, no_file_logging=True,
drop_privileges=False)
controller.upgrade_conf(state_file=state_file, no_backup=True)
if ownership:
change_owner(STATE_FILE_PATH, owner='intelmq', group='intelmq')
def update_database(cls, verbose=False):
bots = {}
runtime_conf = get_bots_settings()
try:
for bot in runtime_conf:
if runtime_conf[bot]["module"] == __name__:
bots[bot] = runtime_conf[bot]["parameters"]["suffix_file"]
except KeyError as e:
sys.exit(
"Database update failed. Your configuration of {0} is missing key {1}."
.format(bot, e))
if not bots:
if verbose:
print(
"Database update skipped. No bots of type {0} present in runtime.conf."
.format(__name__))
sys.exit(0)
# we only need to import now. If there are no asn_lookup bots, this dependency does not need to be installed
try:
session = create_request_session()
url = "https://publicsuffix.org/list/public_suffix_list.dat"
if verbose:
print("Downloading the latest database update...")
response = session.get(url)
if not response.ok:
sys.exit("Database update failed. Server responded: {0}.\n"
"URL: {1}".format(response.status_code, response.url))
except requests.exceptions.RequestException as e:
sys.exit("Database update failed. Connection Error: {0}".format(e))
for database_path in set(bots.values()):
database_dir = pathlib.Path(database_path).parent
database_dir.mkdir(parents=True, exist_ok=True)
with open(database_path, "wb") as database:
database.write(response.content)
if verbose:
print("Database updated. Reloading affected bots.")
ctl = IntelMQController()
for bot in bots.keys():
ctl.bot_reload(bot)
def update_database(cls):
bots = {}
runtime_conf = load_configuration(RUNTIME_CONF_FILE)
try:
for bot in runtime_conf:
if runtime_conf[bot]["module"] == __name__:
bots[bot] = runtime_conf[bot]["parameters"]["database"]
except KeyError as e:
sys.exit(
"Database update failed. Your configuration of {0} is missing key {1}."
.format(bot, e))
if not bots:
print(
"Database update skipped. No bots of type {0} present in runtime.conf."
.format(__name__))
sys.exit(0)
try:
print("Downloading the latest database update...")
session = create_request_session()
response = session.get(
"https://check.torproject.org/exit-addresses")
except requests.exceptions.RequestException as e:
sys.exit("Database update failed. Connection Error: {0}".format(e))
if response.status_code != 200:
sys.exit("Database update failed. Server responded: {0}.\n"
"URL: {1}".format(response.status_code, response.url))
pattern = re.compile(r"ExitAddress ([^\s]+)")
tor_exits = "\n".join(pattern.findall(response.text))
for database_path in set(bots.values()):
database_dir = pathlib.Path(database_path).parent
database_dir.mkdir(parents=True, exist_ok=True)
with open(database_path, "w") as database:
database.write(tor_exits)
print("Database updated. Reloading affected bots.")
ctl = IntelMQController()
for bot in bots.keys():
ctl.bot_reload(bot)
def intelmqsetup(ownership=True, state_file=STATE_FILE_PATH):
if os.geteuid() != 0 and ownership:
sys.exit(
'You need to run this program as root (for setting file ownership)'
)
if not ROOT_DIR:
sys.exit('Not a pip-installation of IntelMQ, nothing to initialize.')
create_dirs = ('%s/file-output' % VAR_STATE_PATH, VAR_RUN_PATH,
DEFAULT_LOGGING_PATH, CONFIG_DIR)
for create_dir in create_dirs:
if not os.path.isdir(create_dir):
os.makedirs(create_dir, mode=0o755, exist_ok=True)
print('Created directory %r.' % create_dir)
example_confs = glob.glob(
pkg_resources.resource_filename('intelmq', 'etc/*.conf'))
for example_conf in example_confs:
fname = os.path.split(example_conf)[-1]
if os.path.exists(os.path.join(CONFIG_DIR, fname)):
print('Not overwriting existing %r with example.' % fname)
else:
shutil.copy(example_conf, CONFIG_DIR)
print('Use example %r.' % fname)
print('Writing BOTS file.')
shutil.copy(pkg_resources.resource_filename('intelmq', 'bots/BOTS'),
BOTS_FILE)
if ownership:
print('Setting intelmq as owner for it\'s directories.')
for obj in (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
VAR_STATE_PATH, VAR_STATE_PATH + 'file-output'):
if getpwuid(os.stat(obj).st_uid).pw_name != 'intelmq':
shutil.chown(obj, user='******')
print('Calling `intelmqctl upgrade-config to update/create state file')
controller = IntelMQController(interactive=False,
no_file_logging=True,
drop_privileges=False)
controller.upgrade_conf(state_file=state_file, no_backup=True)
def update_database(cls, verbose=False):
bots = {}
runtime_conf = get_bots_settings()
try:
for bot in runtime_conf:
if runtime_conf[bot]["module"] == __name__:
bots[bot] = runtime_conf[bot]["parameters"]["tlds_domains_list"]
except KeyError as e:
sys.exit("Database update failed. Your configuration of {0} is missing key {1}.".format(bot, e))
if not bots:
if verbose:
print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__))
sys.exit(0)
try:
session = create_request_session()
url = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
if verbose:
print("Downloading the latest database update...")
response = session.get(url)
if not response.ok:
sys.exit("Database update failed. Server responded: {0}.\n"
"URL: {1}".format(response.status_code, response.url))
except requests.exceptions.RequestException as e:
sys.exit("Database update failed. Connection Error: {0}".format(e))
for database_path in set(bots.values()):
database_dir = pathlib.Path(database_path).parent
database_dir.mkdir(parents=True, exist_ok=True)
with open(database_path, "wb") as database:
database.write(response.content)
if verbose:
print("Database updated. Reloading affected bots.")
ctl = IntelMQController()
for bot in bots.keys():
ctl.bot_reload(bot)
def update_database(cls, verbose=False):
bots = {}
api_token = None
runtime_conf = get_bots_settings()
try:
for bot in runtime_conf:
if runtime_conf[bot]["module"] == __name__:
api_token = runtime_conf[bot]["parameters"]["api_token"]
bots[bot] = runtime_conf[bot]["parameters"]["database"]
except KeyError as e:
sys.exit(
"Database update failed. Your configuration of {0} is missing key {1}."
.format(bot, e))
if not bots:
if verbose:
print(
"Database update skipped. No bots of type {0} present in runtime.conf."
.format(__name__))
sys.exit(0)
try:
if verbose:
print("Downloading the latest database update...")
session = create_request_session()
response = session.get(
"https://api.recordedfuture.com/v2/ip/risklist",
params={
"format": "csv/splunk",
"gzip": "true",
"list": "large"
},
headers={"X-RFToken": api_token})
except requests.exceptions.RequestException as e:
sys.exit("Database update failed. Connection Error: {0}".format(e))
if response.status_code == 401:
sys.exit("Database update failed. Your API token is invalid.")
if response.status_code != 200:
sys.exit("Database update failed. Server responded: {0}.\n"
"URL: {1}".format(response.status_code, response.url))
database_data = None
with tarfile.open(fileobj=io.BytesIO(response.content),
mode='r:gz') as archive:
for member in archive.getmembers():
if "rfiprisk.dat" in member.name:
database_data = archive.extract(member)
break
if not database_data:
sys.exit(
"Database update failed. Could not locate file 'rfiprisk.dat' in the downloaded archive."
)
for database_path in set(bots.values()):
database_dir = pathlib.Path(database_path).parent
database_dir.mkdir(parents=True, exist_ok=True)
with open(database_path, "w") as database:
database.write(database_data)
if verbose:
print("Database updated. Reloading affected bots.")
ctl = IntelMQController()
for bot in bots.keys():
ctl.bot_reload(bot)
def update_database(cls):
bots = {}
runtime_conf = load_configuration(RUNTIME_CONF_FILE)
try:
for bot in runtime_conf:
if runtime_conf[bot]["module"] == __name__:
bots[bot] = runtime_conf[bot]["parameters"]["database"]
except KeyError as e:
sys.exit(
"Database update failed. Your configuration of {0} is missing key {1}."
.format(bot, e))
if not bots:
print(
"Database update skipped. No bots of type {0} present in runtime.conf."
.format(__name__))
sys.exit(0)
# we only need to import now. If there are no asn_lookup bots, this dependency does not need to be installed
if pyasn is None:
raise MissingDependencyError("pyasn")
try:
print("Searching for the latest database update...")
session = create_request_session()
url = "http://archive.routeviews.org/route-views4/bgpdata/"
response = session.get(url)
pattern = re.compile(r"href=\"(\d{4}\.\d{2})/\"")
months = pattern.findall(response.text)
months.sort(reverse=True)
if not months:
sys.exit(
"Database update failed. Couldn't find the latest database update."
)
url += str(months[0]) + "/RIBS/"
response = session.get(url)
pattern = re.compile(r"href=\"(rib\.\d{8}\.\d{4}\.bz2)\"")
days = pattern.findall(response.text)
days.sort(reverse=True)
if not days:
sys.exit(
"Database update failed. Couldn't find the latest database update."
)
print("Downloading the latest database update...")
url += days[0]
response = session.get(url)
if response.status_code != 200:
sys.exit("Database update failed. Server responded: {0}.\n"
"URL: {1}".format(response.status_code, response.url))
except requests.exceptions.RequestException as e:
sys.exit("Database update failed. Connection Error: {0}".format(e))
with bz2.open(io.BytesIO(response.content)) as archive:
print("Parsing the latest database update...")
prefixes = pyasn.mrtx.parse_mrt_file(archive,
print_progress=False,
skip_record_on_error=True)
for database_path in set(bots.values()):
database_dir = pathlib.Path(database_path).parent
database_dir.mkdir(parents=True, exist_ok=True)
pyasn.mrtx.dump_prefixes_to_file(prefixes, database_path)
print("Database updated. Reloading affected bots.")
ctl = IntelMQController()
for bot in bots.keys():
ctl.bot_reload(bot)
def update_database(cls):
bots = {}
license_key = None
runtime_conf = load_configuration(RUNTIME_CONF_FILE)
try:
for bot in runtime_conf:
if runtime_conf[bot]["module"] == __name__:
license_key = runtime_conf[bot]["parameters"][
"license_key"]
bots[bot] = runtime_conf[bot]["parameters"]["database"]
except KeyError as e:
error = "Database update failed. Your configuration of {0} is missing key {1}.".format(
bot, e)
if str(e) == "'license_key'":
error += "\n"
error += "Since December 30, 2019 you need to register for a free license key to access GeoLite2 database.\n"
error += "https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/"
sys.exit(error)
else:
sys.exit(error)
if not bots:
print(
"Database update skipped. No bots of type {0} present in runtime.conf."
.format(__name__))
sys.exit(0)
# we only need to import now, if there are no maxmind_geoip bots, this dependency does not need to be installed
try:
import maxminddb
except ImportError:
raise MissingDependencyError(
'maxminddb',
additional_text="Package maxminddb should be present because it "
"is a dependency for the required geoip2 package.")
try:
print("Downloading the latest database update...")
session = create_request_session()
response = session.get(
"https://download.maxmind.com/app/geoip_download",
params={
"license_key": license_key,
"edition_id": "GeoLite2-City",
"suffix": "tar.gz"
})
except requests.exceptions.RequestException as e:
sys.exit("Database update failed. Connection Error: {0}".format(e))
if response.status_code == 401:
sys.exit("Database update failed. Your license key is invalid.")
if response.status_code != 200:
sys.exit("Database update failed. Server responded: {0}.\n"
"URL: {1}".format(response.status_code, response.url))
database_data = None
try:
with tarfile.open(fileobj=io.BytesIO(response.content),
mode='r:gz') as archive:
for member in archive.getmembers():
if "GeoLite2-City.mmdb" in member.name:
database_data = maxminddb.open_database(
database=archive.extractfile(member),
mode=maxminddb.MODE_FD)
break
except maxminddb.InvalidDatabaseError:
sys.exit("Database update failed. Database file invalid.")
if not database_data:
sys.exit(
"Database update failed. Could not locate file 'GeoLite2-City.mmbd' in the downloaded archive."
)
for database_path in set(bots.values()):
database_dir = pathlib.Path(database_path).parent
database_dir.mkdir(parents=True, exist_ok=True)
with open(database_path, "wb") as database:
database.write(database_data._buffer)
print("Database updated. Reloading affected bots.")
ctl = IntelMQController()
for bot in bots.keys():
ctl.bot_reload(bot)