def setup_filebeat(self):
"""
Creates necessary directory structure, and copies required files, generates a default configuration
"""
if self.stdout:
sys.stdout.write('[+] Creating Filebeat install directory.\n')
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
if self.stdout:
sys.stdout.write('[+] Copying Filebeat to install directory.\n')
utilities.copytree(
os.path.join(const.INSTALL_CACHE, const.FILE_BEAT_DIRECTORY_NAME),
self.install_directory)
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'filebeat', 'filebeat.yml'),
self.install_directory)
if self.stdout:
sys.stdout.write(
'[+] Building configurations and setting up permissions.\n')
beats_config = FileBeatConfigurator(self.install_directory)
beats_config.set_monitor_target_paths(self.monitor_paths)
beats_config.write_config()
utilities.set_permissions_of_file(os.path.join(self.install_directory,
'filebeat.yml'),
unix_permissions_integer=501)
if 'FILEBEAT_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating FileBeat default script path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo FILEBEAT_HOME="{}" >> /etc/dynamite/environment'.format(
self.install_directory),
shell=True)
def _copy_suricata_files_and_directories(self):
if self.stdout:
sys.stdout.write(
'[+] Creating suricata install|configuration|logging directories.\n'
)
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
subprocess.call('mkdir -p {}'.format(self.configuration_directory),
shell=True)
subprocess.call('mkdir -p {}'.format(self.log_directory), shell=True)
try:
os.mkdir(os.path.join(self.configuration_directory, 'rules'))
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'suricata',
'suricata.yaml'),
os.path.join(self.configuration_directory, 'suricata.yaml'))
utilities.copytree(
os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME, 'rules'),
os.path.join(self.configuration_directory, 'rules'))
except Exception as e:
sys.stderr.write(
'[-] Unable to re-create Suricata rules directory: {}\n'.
format(e))
return False
return True
def setup_oinkmaster(self):
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info("Installing Oinkmaster.")
try:
utilities.makedirs(self.install_directory, exist_ok=True)
except Exception as e:
self.logger.error("Failed to create required directory structure.")
self.logger.debug("Failed to create required directory structure; {}".format(e))
raise oinkmaster_exceptions.InstallOinkmasterError(
"Failed to create required directory structure; {}".format(e))
self.logger.info("Copying oinkmaster files.")
try:
utilities.copytree(os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME),
self.install_directory)
except Exception as e:
self.logger.error(
'Failed to copy {} -> {}.'.format(os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME),
self.install_directory))
self.logger.debug(sys.stderr.write('Failed to copy {} -> {}: {}'.format(
os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME), self.install_directory, e)))
raise oinkmaster_exceptions.InstallOinkmasterError(
"General error while copying Oinkmaster from install cache; {}".format(e))
if 'OINKMASTER_HOME' not in open(env_file).read():
self.logger.info('Updating Oinkmaster default home path [{}]'.format(self.install_directory))
subprocess.call('echo OINKMASTER_HOME="{}" >> {}'.format(self.install_directory, env_file),
shell=True)
self.logger.info('PATCHING oinkmaster.conf with emerging-threats URL.')
try:
with open(os.path.join(self.install_directory, 'oinkmaster.conf'), 'a') as f:
f.write('\nurl = http://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz')
except Exception as e:
self.logger.error('Failed to update oinkmaster.conf.')
self.logger.debug('Failed to update oinkmaster.conf: {}.\n'.format(e))
raise oinkmaster_exceptions.InstallOinkmasterError(
"Failed to update oinkmaster configuration file; {}".format(e))
def setup_oinkmaster(self, stdout=False):
try:
os.mkdir(self.install_directory)
except Exception as e:
if 'exists' not in str(e).lower():
return False
if stdout:
sys.stdout.write('[+] Copying oinkmaster files.\n')
try:
utilities.copytree(
os.path.join(const.INSTALL_CACHE,
const.OINKMASTER_DIRECTORY_NAME),
self.install_directory)
except Exception as e:
sys.stderr.write('[-] Failed to copy {} -> {}: {}'.format(
os.path.join(const.INSTALL_CACHE,
const.OINKMASTER_DIRECTORY_NAME),
self.install_directory, e))
return False
if stdout:
sys.stdout.write(
'[+] Updating oinkmaster.conf with emerging-threats URL.\n')
try:
with open(os.path.join(self.install_directory, 'oinkmaster.conf'),
'a') as f:
f.write(
'\nurl = http://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz'
)
except Exception as e:
sys.stderr.write(
'[-] Failed to update oinkmaster.conf: {}.\n'.format(e))
return False
return True
def setup_logstash_elastiflow(self, stdout=False):
if stdout:
sys.stdout.write(
'[+] Creating elastiflow install|configuration directories.\n')
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
if stdout:
sys.stdout.write('[+] Copying elastiflow configurations\n')
utilities.copytree(
os.path.join(const.DEFAULT_CONFIGS, 'logstash', 'zeek'),
self.install_directory)
utilities.set_ownership_of_file(self.install_directory,
user='******',
group='dynamite')
if 'ELASTIFLOW_DICT_PATH' not in open(
'/etc/dynamite/environment').read():
dict_path = os.path.join(self.install_directory, 'dictionaries')
if stdout:
sys.stdout.write(
'[+] Updating Elastiflow dictionary configuration path [{}]\n'
.format(dict_path))
subprocess.call(
'echo ELASTIFLOW_DICT_PATH="{}" >> /etc/dynamite/environment'.
format(dict_path),
shell=True)
if 'ELASTIFLOW_TEMPLATE_PATH' not in open(
'/etc/dynamite/environment').read():
template_path = os.path.join(self.install_directory, 'templates')
if stdout:
sys.stdout.write(
'[+] Updating Elastiflow template configuration path [{}]\n'
.format(template_path))
subprocess.call(
'echo ELASTIFLOW_TEMPLATE_PATH="{}" >> /etc/dynamite/environment'
.format(template_path),
shell=True)
if 'ELASTIFLOW_GEOIP_DB_PATH' not in open(
'/etc/dynamite/environment').read():
geo_path = os.path.join(self.install_directory, 'geoipdbs')
if stdout:
sys.stdout.write(
'[+] Updating Elastiflow geodb configuration path [{}]\n'.
format(geo_path))
subprocess.call(
'echo ELASTIFLOW_GEOIP_DB_PATH="{}" >> /etc/dynamite/environment'
.format(geo_path),
shell=True)
if 'ELASTIFLOW_DEFINITION_PATH' not in open(
'/etc/dynamite/environment').read():
def_path = os.path.join(self.install_directory, 'definitions')
if stdout:
sys.stdout.write(
'[+] Updating Elastiflow definitions configuration path [{}]\n'
.format(def_path))
subprocess.call(
'echo ELASTIFLOW_DEFINITION_PATH="{}" >> /etc/dynamite/environment'
.format(def_path),
shell=True)
ElastiflowConfigurator().write_environment_variables()
def setup(self):
utilities.makedirs(self.install_directory)
self.create_update_oinkmaster_environment_variables()
utilities.copytree(f'{const.INSTALL_CACHE}/{self.local_mirror_root}',
self.install_directory)
with open(os.path.join(self.install_directory, 'oinkmaster.conf'),
'a') as f:
f.write(f'\nurl = {const.EMERGING_THREATS_OPEN}')
def _copy_elasticsearch_files_and_directories(self):
config_paths = [
'config/elasticsearch.yml', 'config/jvm.options',
'config/log4j2.properties'
]
install_paths = ['bin/', 'lib/', 'logs/', 'modules/', 'plugins/']
path = None
try:
for path in config_paths:
self.logger.debug('Copying {} -> {}'.format(
os.path.join(
const.INSTALL_CACHE,
'{}/{}'.format(const.ELASTICSEARCH_DIRECTORY_NAME,
path)), self.configuration_directory))
try:
shutil.copy(
os.path.join(
const.INSTALL_CACHE,
'{}/{}'.format(const.ELASTICSEARCH_DIRECTORY_NAME,
path)),
self.configuration_directory)
except shutil.Error:
self.logger.warning(
'{} already exists at this path.'.format(path))
except Exception as e:
self.logger.error(
"General error while attempting to copy {} to {}.".format(
path, self.configuration_directory))
self.logger.debug(
"General error while attempting to copy {} to {}; {}".format(
path, self.configuration_directory, e))
raise elastic_exceptions.InstallElasticsearchError(
"General error while attempting to copy {} to {}; {}".format(
path, self.configuration_directory, e))
try:
for path in install_paths:
src_install_path = os.path.join(
const.INSTALL_CACHE, const.ELASTICSEARCH_DIRECTORY_NAME,
path)
dst_install_path = os.path.join(self.install_directory, path)
self.logger.debug('Copying {} -> {}'.format(
src_install_path, dst_install_path))
try:
utilities.makedirs(dst_install_path, exist_ok=True)
utilities.copytree(src_install_path, dst_install_path)
except shutil.Error:
self.logger.warning(
'{} already exists at this path.'.format(path))
except Exception as e:
self.logger.error(
"General error while attempting to copy {} to {}.".format(
path, self.install_directory))
self.logger.debug(
"General error while attempting to copy {} to {}; {}".format(
path, self.install_directory, e))
raise elastic_exceptions.InstallElasticsearchError(
"General error while attempting to copy {} to {}; {}".format(
path, self.install_directory, e))
def setup_logstash_synesis(self, stdout=False):
if stdout:
sys.stdout.write(
'[+] Creating synesis install|configuration directories.\n')
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
if stdout:
sys.stdout.write('[+] Copying synesis configurations\n')
utilities.copytree(
os.path.join(const.DEFAULT_CONFIGS, 'logstash', 'suricata'),
self.install_directory)
utilities.set_ownership_of_file(self.install_directory,
user='******',
group='dynamite')
if 'SYNLITE_SURICATA_DICT_PATH' not in open(
'/etc/dynamite/environment').read():
dict_path = os.path.join(self.install_directory, 'dictionaries')
if stdout:
sys.stdout.write(
'[+] Updating Synesis dictionary configuration path [{}]\n'
.format(dict_path))
subprocess.call(
'echo SYNLITE_SURICATA_DICT_PATH="{}" >> /etc/dynamite/environment'
.format(dict_path),
shell=True)
if 'SYNLITE_SURICATA_TEMPLATE_PATH' not in open(
'/etc/dynamite/environment').read():
template_path = os.path.join(self.install_directory, 'templates')
if stdout:
sys.stdout.write(
'[+] Updating Synesis template configuration path [{}]\n'.
format(template_path))
subprocess.call(
'echo SYNLITE_SURICATA_TEMPLATE_PATH="{}" >> /etc/dynamite/environment'
.format(template_path),
shell=True)
if 'SYNLITE_SURICATA_GEOIP_DB_PATH' not in open(
'/etc/dynamite/environment').read():
geo_path = os.path.join(self.install_directory, 'geoipdbs')
if stdout:
sys.stdout.write(
'[+] Updating Synesis geodb configuration path [{}]\n'.
format(geo_path))
subprocess.call(
'echo SYNLITE_SURICATA_GEOIP_DB_PATH="{}" >> /etc/dynamite/environment'
.format(geo_path),
shell=True)
SynesisConfigurator().write_environment_variables()
def _copy_suricata_files_and_directories(self):
self.logger.info('Creating Suricata installation, configuration, and logging directories.')
try:
utilities.makedirs(self.install_directory, exist_ok=True)
utilities.makedirs(self.configuration_directory, exist_ok=True)
utilities.makedirs(self.log_directory, exist_ok=True)
except Exception as e:
self.logger.error('Failed to create required directory structure.')
self.logger.debug("Failed to create required directory structure; {}".format(e))
raise suricata_exceptions.InstallSuricataError(
"Failed to create required directory structure; {}".format(e))
try:
utilities.makedirs(os.path.join(self.configuration_directory, 'rules'), exist_ok=True)
except Exception as e:
self.logger.error('Unable to re-create Suricata rules directory.')
self.logger.debug('Unable to re-create Suricata rules directory: {}'.format(e))
try:
shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'suricata', 'suricata.yaml'),
os.path.join(self.configuration_directory, 'suricata.yaml'))
except Exception as e:
self.logger.error("General error while attempting to copy {} to {}.".format(
os.path.join(const.DEFAULT_CONFIGS, 'suricata', 'suricata.yaml'), self.configuration_directory))
self.logger.debug("General error while attempting to copy {} to {}; {}".format(
os.path.join(const.DEFAULT_CONFIGS, 'suricata', 'suricata.yaml'), self.configuration_directory, e))
raise suricata_exceptions.InstallSuricataError(
"General error while attempting to copy {} to {}; {}".format(
os.path.join(const.DEFAULT_CONFIGS, 'suricata', 'suricata.yaml'), self.configuration_directory, e))
try:
utilities.copytree(os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME, 'rules'),
os.path.join(self.configuration_directory, 'rules'))
except Exception as e:
self.logger.error("General error while attempting to copy {} to {}.".format(
os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME, 'rules'),
os.path.join(self.configuration_directory, 'rules')))
self.logger.debug("General error while attempting to copy {} to {}; {}".format(
os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME, 'rules'),
os.path.join(self.configuration_directory, 'rules'), e))
raise suricata_exceptions.InstallSuricataError(
"General error while attempting to copy {} to {}; {}".format(
os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME, 'rules'),
os.path.join(self.configuration_directory, 'rules'), e))
def setup_oinkmaster(self):
try:
os.mkdir(self.install_directory)
except Exception as e:
if 'exists' not in str(e).lower():
return False
if self.stdout:
sys.stdout.write('[+] Copying oinkmaster files.\n')
try:
utilities.copytree(
os.path.join(const.INSTALL_CACHE,
const.OINKMASTER_DIRECTORY_NAME),
self.install_directory)
except Exception as e:
sys.stderr.write('[-] Failed to copy {} -> {}: {}'.format(
os.path.join(const.INSTALL_CACHE,
const.OINKMASTER_DIRECTORY_NAME),
self.install_directory, e))
return False
if 'OINKMASTER_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Oinkmaster default home path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo OINKMASTER_HOME="{}" >> /etc/dynamite/environment'.
format(self.install_directory),
shell=True)
if self.stdout:
sys.stdout.write(
'[+] Updating oinkmaster.conf with emerging-threats URL.\n')
try:
with open(os.path.join(self.install_directory, 'oinkmaster.conf'),
'a') as f:
f.write(
'\nurl = http://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz'
)
except Exception as e:
sys.stderr.write(
'[-] Failed to update oinkmaster.conf: {}.\n'.format(e))
return False
return True
def copy_file_or_directory_to_destination(self, file_or_dir: str, destination_file_or_dir: str) -> None:
"""
Copy a file or directory to another file or directory
Args:
file_or_dir: The file or directory to copy
destination_file_or_dir: The file or directory destination
Returns:
None
"""
file_or_dir = file_or_dir.rstrip('/')
destination_location = f'{destination_file_or_dir}/{os.path.basename(file_or_dir)}'
if os.path.isdir(file_or_dir):
utilities.makedirs(destination_location, exist_ok=True)
self.logger.debug(f'Creating directory: {destination_location}')
try:
self.logger.debug(f'Copying directory {file_or_dir} -> {destination_location}')
utilities.copytree(file_or_dir, destination_location)
except shutil.Error as e:
if 'exist' in str(e):
self.logger.warning(f'{destination_file_or_dir} directory already exists. Skipping.')
else:
raise e
else:
try:
self.logger.debug(f'Copying file {file_or_dir} -> {destination_file_or_dir}')
shutil.copy(file_or_dir, destination_file_or_dir)
except FileNotFoundError:
parent_directory = os.path.dirname(destination_file_or_dir)
self.logger.debug(f'Creating parent directory: {parent_directory}')
utilities.makedirs(parent_directory)
shutil.copy(file_or_dir, destination_file_or_dir)
except shutil.Error as e:
if 'exist' in str(e):
self.logger.warning(f'{destination_file_or_dir} file already exists. Skipping.')
else:
raise e
def setup_dynamite_sdk(self):
"""
Sets up sdk files; and installs globally
"""
if self.stdout:
sys.stdout.write('[+] Copying DynamiteSDK into lab environment.\n')
sys.stdout.flush()
subprocess.call('mkdir -p {}'.format(self.notebook_home), shell=True)
if 'NOTEBOOK_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Notebook home path [{}]\n'.format(
self.notebook_home))
subprocess.call(
'echo NOTEBOOK_HOME="{}" >> /etc/dynamite/environment'.
format(self.notebook_home),
shell=True)
subprocess.call('mkdir -p {}'.format(self.configuration_directory),
shell=True)
if 'DYNAMITE_LAB_CONFIG' not in open(
'/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Dynamite Lab Config path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo DYNAMITE_LAB_CONFIG="{}" >> /etc/dynamite/environment'.
format(self.configuration_directory),
shell=True)
sdk_install_cache = os.path.join(const.INSTALL_CACHE,
const.DYNAMITE_SDK_DIRECTORY_NAME)
utilities.copytree(os.path.join(sdk_install_cache, 'notebooks'),
self.notebook_home)
shutil.copy(
os.path.join(sdk_install_cache, 'dynamite_sdk',
'config.cfg.example'),
os.path.join(self.configuration_directory, 'config.cfg'))
utilities.set_ownership_of_file(self.notebook_home,
user='******',
group='jupyter')
if self.stdout:
sys.stdout.write(
'[+] Installing dynamite-sdk-lite (https://github.com/DynamiteAI/dynamite-sdk-lite)\n'
)
sys.stdout.write(
'[+] Depending on your distribution it may take some time to install all requirements.\n'
)
sys.stdout.flush()
if self.verbose:
p = subprocess.Popen(['python3', 'setup.py', 'install'],
cwd=sdk_install_cache)
else:
p = subprocess.Popen(['python3', 'setup.py', 'install'],
cwd=sdk_install_cache,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p.communicate()
dynamite_sdk_config = DynamiteLabConfigurator(
configuration_directory=self.configuration_directory)
dynamite_sdk_config.elasticsearch_url = 'http://{}:{}'.format(
self.elasticsearch_host, self.elasticsearch_port)
dynamite_sdk_config.elasticsearch_user = '******'
dynamite_sdk_config.elasticsearch_password = self.elasticsearch_password
dynamite_sdk_config.write_config()
def setup_filebeat(self):
"""
Creates necessary directory structure, and copies required files, generates a default configuration
"""
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info('Creating FileBeat install directory.')
utilities.makedirs(self.install_directory, exist_ok=True)
self.logger.info('Copying FileBeat to install directory.')
try:
utilities.copytree(
os.path.join(const.INSTALL_CACHE,
const.FILE_BEAT_DIRECTORY_NAME),
self.install_directory)
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'filebeat',
'filebeat.yml'), self.install_directory)
except Exception as e:
self.logger.error(
"General error occurred while copying Filebeat configs.")
self.logger.debug(
"General error occurred while copying Filebeat configs; {}".
format(e))
raise filebeat_exceptions.InstallFilebeatError(
"General error occurred while copying Filebeat configs; {}".
format(e))
self.logger.info("Building configurations and setting up permissions.")
try:
beats_config = filebeat_configs.ConfigManager(
self.install_directory)
except filebeat_exceptions.ReadFilebeatConfigError:
self.logger.error("Failed to read Filebeat configuration.")
raise filebeat_exceptions.InstallFilebeatError(
"Failed to read Filebeat configuration.")
beats_config.set_monitor_target_paths(self.monitor_paths)
beats_config.set_agent_tag(self.agent_tag)
if (self.kafka_password
or self.kafka_username) and not self.kafka_topic:
self.logger.error(
"You have specified Kafka config options without specifying a Kafka topic."
)
raise filebeat_exceptions.InstallFilebeatError(
"You have specified Kafka config options without specifying a Kafka topic."
)
if self.kafka_topic:
self.logger.warning(
"You have enabled the Agent's Kafka output which does integrate natively with Dynamite "
"Monitor/LogStash component. You will have to bring your own broker. Happy Hacking!"
)
time.sleep(2)
beats_config.set_kafka_targets(target_hosts=self.targets,
topic=self.kafka_topic,
username=self.kafka_username,
password=self.kafka_password)
# setup example upstream LogStash example, just in case you want to configure later
beats_config.set_logstash_targets(target_hosts=['localhost:5601'])
beats_config.enable_kafka_output()
else:
# setup example upstream Kafka example, just in case you want to configure later
beats_config.set_kafka_targets(target_hosts=['localhost:9092'],
topic='dynamite-nsm-events')
beats_config.enable_logstash_output()
beats_config.set_logstash_targets(self.targets)
try:
beats_config.write_config()
except filebeat_exceptions.WriteFilebeatConfigError:
self.logger.error("Failed to write filebeat configuration.")
raise filebeat_exceptions.InstallFilebeatError(
"Failed to write filebeat configuration.")
try:
utilities.set_permissions_of_file(os.path.join(
self.install_directory, 'filebeat.yml'),
unix_permissions_integer=501)
except Exception as e:
self.logger.error(
"Failed to set permissions of filebeat.yml file.")
self.logger.debug(
"Failed to set permissions of filebeat.yml file; {}".format(e))
filebeat_exceptions.InstallFilebeatError(
"Failed to set permissions of filebeat.yml file; {}".format(e))
try:
with open(env_file) as env_f:
if 'FILEBEAT_HOME' not in env_f.read():
self.logger.info(
'Updating FileBeat default script path [{}]'.format(
self.install_directory))
subprocess.call('echo FILEBEAT_HOME="{}" >> {}'.format(
self.install_directory, env_file),
shell=True)
except Exception as e:
self.logger.error(
"General error occurred while attempting to install FileBeat.")
self.logger.debug(
"General error occurred while attempting to install FileBeat; {}"
.format(e))
raise filebeat_exceptions.InstallFilebeatError(
"General error occurred while attempting to install FileBeat; {}"
.format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise filebeat_exceptions.InstallFilebeatError(
"Could not find systemctl.")
self.logger.info("Installing Filebeat systemd service.")
if not sysctl.install_and_enable(
os.path.join(const.DEFAULT_CONFIGS, 'systemd',
'filebeat.service')):
raise filebeat_exceptions.InstallFilebeatError(
"Failed to install Filebeat systemd service.")
sysctl.install_and_enable(
os.path.join(const.DEFAULT_CONFIGS, 'systemd',
'dynamite-agent.target'))
zeek_logs = None
zeek_home = self.environ.get('ZEEK_HOME')
suricata_logs = os.path.join(const.LOG_PATH, 'suricata')
if zeek_home:
zeek_logs = os.path.join(zeek_home, 'logs', 'current')
try:
self.logger.info('Patching Zeek/Suricata modules.')
beats_config.patch_modules(zeek_log_directory=zeek_logs,
suricata_log_directory=suricata_logs)
except filebeat_exceptions.WriteFilebeatModuleError:
self.logger.error('Could not patch Zeek/Suricata modules.')
raise filebeat_exceptions.InstallFilebeatError(
"Could not patch Zeek/Suricata modules.")
def setup_logstash_elastiflow(self):
"""
Create required environmental variables; copy configurations to various directories.
"""
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info(
'Creating ElastiFlow installation and configuration directories.')
utilities.makedirs(self.install_directory, exist_ok=True)
self.logger.info('Copying ElastiFlow configurations.')
utilities.copytree(
os.path.join(const.DEFAULT_CONFIGS, 'logstash', 'zeek'),
self.install_directory)
utilities.set_ownership_of_file(self.install_directory,
user='******',
group='dynamite')
try:
with open(env_file) as env_f:
env_str = env_f.read()
if 'ELASTIFLOW_DICT_PATH' not in env_str:
dict_path = os.path.join(self.install_directory,
'dictionaries')
self.logger.info(
'Updating ElastiFlow dictionary configuration path [{}]'
.format(dict_path))
subprocess.call(
'echo ELASTIFLOW_DICT_PATH="{}" >> {}'.format(
dict_path, env_file),
shell=True)
if 'ELASTIFLOW_TEMPLATE_PATH' not in env_str:
template_path = os.path.join(self.install_directory,
'templates')
self.logger.info(
'Updating ElastiFlow template configuration path [{}]'.
format(template_path))
subprocess.call(
'echo ELASTIFLOW_TEMPLATE_PATH="{}" >> {}'.format(
template_path, env_file),
shell=True)
if 'ELASTIFLOW_GEOIP_DB_PATH' not in env_str:
geo_path = os.path.join(self.install_directory, 'geoipdbs')
self.logger.info(
'Updating ElastiFlow GeoDBs configuration path [{}]'.
format(geo_path))
subprocess.call(
'echo ELASTIFLOW_GEOIP_DB_PATH="{}" >> {}'.format(
geo_path, env_file),
shell=True)
if 'ELASTIFLOW_DEFINITION_PATH' not in env_str:
def_path = os.path.join(self.install_directory,
'definitions')
self.logger.info(
'Updating ElastiFlow definitions configuration path [{}]'
.format(def_path))
subprocess.call(
'echo ELASTIFLOW_DEFINITION_PATH="{}" >> {}'.format(
def_path, env_file),
shell=True)
except Exception as e:
self.logger.error(
'Failed to read ElastiFlow environment variables.')
self.logger.debug(
"Failed to read ElastiFlow environment variables; {}".format(
e))
raise elastiflow_exceptions.InstallElastiflowError(
"Failed to read ElastiFlow environment variables; {}".format(
e))
try:
elastiflow_config.ConfigManager().write_environment_variables()
except (elastiflow_exceptions.ReadElastiflowConfigError,
elastiflow_exceptions.WriteElastiflowConfigError):
self.logger.error(
'Failed to read/write ElastiFlow environment variables.')
raise elastiflow_exceptions.InstallElastiflowError(
"Could not read/write ElastiFlow environment variables.")
def setup_dynamite_sdk(self):
"""
Sets up sdk files; and installs globally
"""
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info('Copying DynamiteSDK into lab environment.')
try:
utilities.makedirs(self.notebook_home, exist_ok=True)
utilities.makedirs(self.configuration_directory, exist_ok=True)
except Exception as e:
self.logger.error(
'General error occurred while attempting to create root directories.'
)
self.logger.debug(
"General error occurred while attempting to create root directories; {}"
.format(e))
raise lab_exceptions.InstallLabError(
"General error occurred while attempting to create root directories; {}"
.format(e))
try:
with open(env_file) as env_f:
env_str = env_f.read()
if 'NOTEBOOK_HOME' not in env_str:
self.logger.info('Updating Notebook home path [{}]'.format(
self.notebook_home))
subprocess.call('echo NOTEBOOK_HOME="{}" >> {}'.format(
self.notebook_home, env_file),
shell=True)
if 'DYNAMITE_LAB_CONFIG' not in env_str:
self.logger.info(
'Updating Dynamite Lab Config path [{}]'.format(
self.configuration_directory))
subprocess.call(
'echo DYNAMITE_LAB_CONFIG="{}" >> {}'.format(
self.configuration_directory, env_file),
shell=True)
except IOError:
self.logger.error(
"Failed to open {} for reading.".format(env_file))
raise lab_exceptions.InstallLabError(
"Failed to open {} for reading.".format(env_file))
except Exception as e:
self.logger.error(
"General error while creating environment variables in {}.")
self.logger.debug(
"General error while creating environment variables in {}; {}".
format(env_file, e))
raise lab_exceptions.InstallLabError(
"General error while creating environment variables in {}; {}".
format(env_file, e))
sdk_install_cache = os.path.join(const.INSTALL_CACHE,
const.DYNAMITE_SDK_DIRECTORY_NAME)
utilities.copytree(os.path.join(sdk_install_cache, 'notebooks'),
self.notebook_home)
shutil.copy(
os.path.join(sdk_install_cache, 'dynamite_sdk',
'config.cfg.example'),
os.path.join(self.configuration_directory, 'config.cfg'))
try:
utilities.set_ownership_of_file(self.notebook_home,
user='******',
group='jupyter')
except Exception as e:
raise lab_exceptions.InstallLabError(
"General error occurred while attempting to set permissions on root directories; {}"
.format(e))
self.logger.info(
'Installing dynamite-sdk-lite (https://github.com/DynamiteAI/dynamite-sdk-lite), '
'Depending on your distribution it may take some time to install all requirements.'
)
if self.verbose:
p = subprocess.Popen(['python3', 'setup.py', 'install'],
cwd=sdk_install_cache)
else:
p = subprocess.Popen(['python3', 'setup.py', 'install'],
cwd=sdk_install_cache,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
p.communicate()
except Exception as e:
self.logger.error(
"General error occurred while installing DynamiteSDK.")
self.logger.debug(
"General error occurred while installing DynamiteSDK; {}".
format(e))
raise lab_exceptions.InstallLabError(
"General error occurred while installing DynamiteSDK; {}".
format(e))
if p.returncode != 0:
self.logger.error(
"DynamiteSDK install process returned non-zero exit-code: {}".
format(p.returncode))
raise lab_exceptions.InstallLabError(
"DynamiteSDK install process returned non-zero exit-code: {}".
format(p.returncode))
try:
dynamite_sdk_config = lab_configs.ConfigManager(
configuration_directory=self.configuration_directory)
except lab_exceptions.ReadLabConfigError:
self.logger.error("Failed to read DynamiteSDK config.cfg.")
raise lab_exceptions.InstallLabError(
"Failed to read DynamiteSDK config.cfg.")
dynamite_sdk_config.elasticsearch_url = 'http://{}:{}'.format(
self.elasticsearch_host, self.elasticsearch_port)
dynamite_sdk_config.elasticsearch_user = '******'
dynamite_sdk_config.elasticsearch_password = self.elasticsearch_password
try:
dynamite_sdk_config.write_config()
except lab_exceptions.WriteLabConfigError:
self.logger.error("Failed to write DynamiteSDK config.cfg.")
raise lab_exceptions.InstallLabError(
"Failed to write DynamiteSDK config.cfg.")
def setup_dynamite_zeek_scripts(self):
"""
Installs and enables extra dynamite Zeek scripts
:return: True if zeek scripts were successfully installed
"""
scripts = ''
redefs = ''
if self.stdout:
sys.stdout.write('[+] Setting up Zeek scripts.\n')
install_cache_extra_scripts_path = \
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'dynamite_extra_scripts')
if not os.path.exists(install_cache_extra_scripts_path):
sys.stderr.write(
'[-] dynamite_extra_scripts not found in install_cache.\n')
sys.stderr.flush()
return False
try:
os.mkdir(
os.path.join(self.configuration_directory,
'dynamite_extra_scripts'))
except OSError:
pass
if self.stdout:
sys.stdout.write('[+] Installing third-party Zeek scripts.\n')
extra_scripts_path = os.path.join(self.configuration_directory,
'dynamite_extra_scripts')
utilities.copytree(install_cache_extra_scripts_path,
extra_scripts_path)
with open(
os.path.join(self.configuration_directory, 'site',
'local.bro'), 'r') as rf:
for line in rf.readlines():
if '@load' in line:
scripts += line.strip() + '\n'
elif 'redef' in line:
redefs += line.strip() + '\n'
with open(
os.path.join(self.configuration_directory, 'site',
'local.bro'), 'w') as wf:
extra_script_install_path = os.path.join(
self.configuration_directory, 'dynamite_extra_scripts')
wf.write(scripts)
for script_dir in os.listdir(extra_script_install_path):
wf.write('@load {}\n'.format(
os.path.join(extra_script_install_path, script_dir)))
wf.write(redefs)
if self.stdout:
sys.stdout.write('[+] Disabling unneeded Zeek scripts.\n')
# Disable Unneeded Zeek scripts
script_config = ZeekScriptConfigurator()
script_config.disable_script('protocols/ftp/detect')
script_config.disable_script('protocols/ftp/software')
script_config.disable_script('protocols/ftp/detect-bruteforcing')
script_config.disable_script('protocols/dns/detect-external-names')
script_config.disable_script('protocols/http/detect-sqli')
script_config.disable_script('protocols/http/detect-webapps')
script_config.disable_script('protocols/krb/ticket-logging')
script_config.disable_script('protocols/rdp/indicate_ssl')
script_config.disable_script('protocols/smb/log-cmds')
script_config.disable_script('protocols/smtp/blocklists')
script_config.disable_script('protocols/smtp/detect-suspicious-orig')
script_config.disable_script('protocols/smtp/entities-excerpt')
script_config.disable_script('protocols/smtp/blocklists')
script_config.disable_script('protocols/smtp/software')
script_config.disable_script('protocols/ssh/detect-bruteforcing')
script_config.disable_script('protocols/ssh/geo-data')
script_config.disable_script('protocols/ssh/interesting-hostnames')
script_config.disable_script('protocols/ssh/software')
script_config.disable_script('protocols/ssl/expiring-certs')
script_config.disable_script('protocols/ssl/extract-certs-pem')
script_config.disable_script('protocols/ssl/heartbleed')
script_config.disable_script('protocols/ssl/known-certs')
script_config.disable_script('protocols/ssl/notary')
script_config.disable_script('protocols/ssl/validate-ocsp')
script_config.disable_script('protocols/ssl/validate-sct')
script_config.disable_script('protocols/ssl/weak-keys')
script_config.disable_script('frameworks/dpd/detect-protocols')
script_config.disable_script('frameworks/dpd/packet-segment-logging')
script_config.disable_script('frameworks/files/detect-MHR')
script_config.disable_script('frameworks/files/entropy-test-all-files')
script_config.disable_script('frameworks/files/extract-all-files')
script_config.disable_script('frameworks/files/hash-all-files')
script_config.disable_script(
'policy/frameworks/notice/extend-email/hostnames')
script_config.write_config()
return True
def setup_dynamite_zeek_scripts(self):
"""
Installs and enables extra dynamite Zeek scripts
"""
scripts = ''
redefs = ''
self.logger.info('Setting up Zeek scripts.')
install_cache_extra_scripts_path = \
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'dynamite_extra_scripts')
if not os.path.exists(install_cache_extra_scripts_path):
self.logger.error('dynamite_extra_scripts not found in install_cache.')
raise zeek_exceptions.InstallZeekError(
"Third party scripts could not be installed; could not locate {}".format(
install_cache_extra_scripts_path))
try:
utilities.makedirs(os.path.join(self.configuration_directory, 'dynamite_extra_scripts'), exist_ok=True)
except Exception as e:
self.logger.error('General error occurred while creating dynamite_extra_scripts directory.')
self.logger.debug("General error occurred while creating dynamite_extra_scripts directory; {}".format(e))
zeek_exceptions.InstallZeekError(
"General error occurred while creating dynamite_extra_scripts directory; {}".format(e))
self.logger.info("Installing third-party Zeek scripts.")
extra_scripts_path = os.path.join(self.configuration_directory, 'dynamite_extra_scripts')
try:
utilities.copytree(install_cache_extra_scripts_path, extra_scripts_path)
except Exception as e:
self.logger.error("General error occurred while copying files to dynamite_extra_scripts directory.")
self.logger.debug(
"General error occurred while copying files to dynamite_extra_scripts directory; {}".format(e))
zeek_exceptions.InstallZeekError(
"General error occurred while copying files to dynamite_extra_scripts directory; {}".format(e))
zeek_site_local_path = os.path.join(self.configuration_directory, 'site', 'local.zeek')
try:
with open(zeek_site_local_path, 'r') as rf:
for line in rf.readlines():
if '@load' in line:
scripts += line.strip() + '\n'
elif 'redef' in line:
redefs += line.strip() + '\n'
except Exception as e:
self.logger.error("General error occurred while reading {}.".format(e))
self.logger.debug("General error occurred while reading {}; {}".format(zeek_site_local_path, e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while reading {}; {}".format(zeek_site_local_path, e))
try:
with open(zeek_site_local_path, 'w') as wf:
extra_script_install_path = os.path.join(self.configuration_directory, 'dynamite_extra_scripts')
wf.write(scripts)
for script_dir in os.listdir(extra_script_install_path):
wf.write('@load {}\n'.format(os.path.join(extra_script_install_path, script_dir)))
wf.write(redefs)
except Exception as e:
self.logger.error("General error occurred while writing {}.".format(e))
self.logger.debug("General error occurred while writing {}; {}".format(zeek_site_local_path, e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while writing {}; {}".format(zeek_site_local_path, e)
)
self.setup_zeek_af_packet_plugin()
self.setup_zeek_community_id_plugin()
self.logger.info('Disabling unneeded Zeek scripts.')
# Disable Unneeded Zeek scripts
try:
script_config = zeek_configs.ScriptConfigManager(self.configuration_directory)
except zeek_exceptions.ReadsZeekConfigError:
self.logger.error('Could not read Zeek script configuration.')
raise zeek_exceptions.InstallZeekError("Could not read Zeek script configuration.")
try:
self.logger.debug('Disabling Zeek Script: protocols/ftp/detect')
script_config.disable_script('protocols/ftp/detect')
self.logger.debug('Disabling Zeek Script: protocols/ftp/software')
script_config.disable_script('protocols/ftp/software')
self.logger.debug('Disabling Zeek Script: protocols/ftp/detect-bruteforcing')
script_config.disable_script('protocols/ftp/detect-bruteforcing')
self.logger.debug('Disabling Zeek Script: protocols/dns/detect-external-names')
script_config.disable_script('protocols/dns/detect-external-names')
self.logger.debug('Disabling Zeek Script: protocols/http/detect-sqli')
script_config.disable_script('protocols/http/detect-sqli')
self.logger.debug('Disabling Zeek Script: protocols/http/detect-webapps')
script_config.disable_script('protocols/http/detect-webapps')
self.logger.debug('Disabling Zeek Script: protocols/krb/ticket-logging')
script_config.disable_script('protocols/krb/ticket-logging')
self.logger.debug('Disabling Zeek Script: protocols/rdp/indicate_ssl')
script_config.disable_script('protocols/rdp/indicate_ssl')
self.logger.debug('Disabling Zeek Script: protocols/smb/log-cmds')
script_config.disable_script('protocols/smb/log-cmds')
self.logger.debug('Disabling Zeek Script: protocols/smtp/blocklists')
script_config.disable_script('protocols/smtp/blocklists')
self.logger.debug('Disabling Zeek Script: protocols/smtp/detect-suspicious-orig')
script_config.disable_script('protocols/smtp/detect-suspicious-orig')
self.logger.debug('Disabling Zeek Script: protocols/smtp/entities-excerpt')
script_config.disable_script('protocols/smtp/entities-excerpt')
self.logger.debug('Disabling Zeek Script: protocols/smtp/blocklists')
script_config.disable_script('protocols/smtp/blocklists')
self.logger.debug('Disabling Zeek Script: protocols/smtp/software')
script_config.disable_script('protocols/smtp/software')
self.logger.debug('Disabling Zeek Script: protocols/ssh/detect-bruteforcing')
script_config.disable_script('protocols/ssh/detect-bruteforcing')
self.logger.debug('Disabling Zeek Script: protocols/ssh/geo-data')
script_config.disable_script('protocols/ssh/geo-data')
self.logger.debug('Disabling Zeek Script: protocols/ssh/interesting-hostnames')
script_config.disable_script('protocols/ssh/interesting-hostnames')
self.logger.debug('Disabling Zeek Script: protocols/ssh/software')
script_config.disable_script('protocols/ssh/software')
self.logger.debug('Disabling Zeek Script: protocols/ssl/expiring-certs')
script_config.disable_script('protocols/ssl/expiring-certs')
self.logger.debug('Disabling Zeek Script: protocols/ssl/extract-certs-pem')
script_config.disable_script('protocols/ssl/extract-certs-pem')
self.logger.debug('Disabling Zeek Script: protocols/ssl/heartbleed')
script_config.disable_script('protocols/ssl/heartbleed')
self.logger.debug('Disabling Zeek Script: protocols/ssl/known-certs')
script_config.disable_script('protocols/ssl/known-certs')
self.logger.debug('Disabling Zeek Script: protocols/ssl/notary')
script_config.disable_script('protocols/ssl/notary')
self.logger.debug('Disabling Zeek Script: protocols/ssl/validate-ocsp')
script_config.disable_script('protocols/ssl/validate-ocsp')
self.logger.debug('Disabling Zeek Script: protocols/ssl/validate-sct')
script_config.disable_script('protocols/ssl/validate-sct')
self.logger.debug('Disabling Zeek Script: protocols/ssl/weak-keys')
script_config.disable_script('protocols/ssl/weak-keys')
self.logger.debug('Disabling Zeek Script: frameworks/dpd/detect-protocols')
script_config.disable_script('frameworks/dpd/detect-protocols')
self.logger.debug('Disabling Zeek Script: frameworks/dpd/packet-segment-logging')
script_config.disable_script('frameworks/dpd/packet-segment-logging')
self.logger.debug('Disabling Zeek Script: frameworks/files/detect-MHR')
script_config.disable_script('frameworks/files/detect-MHR')
self.logger.debug('Disabling Zeek Script: frameworks/files/entropy-test-all-files')
script_config.disable_script('frameworks/files/entropy-test-all-files')
self.logger.debug('Disabling Zeek Script: frameworks/files/extract-all-files')
script_config.disable_script('frameworks/files/extract-all-files')
self.logger.debug('Disabling Zeek Script: frameworks/files/hash-all-files')
script_config.disable_script('frameworks/files/hash-all-files')
self.logger.debug('Disabling Zeek Script: policy/frameworks/notice/extend-email/hostnames')
script_config.disable_script('policy/frameworks/notice/extend-email/hostnames')
except zeek_exceptions.ZeekScriptNotFoundError:
self.logger.error('Could not disable one or more Zeek scripts.')
raise zeek_exceptions.InstallZeekError("Could not disable one or more Zeek scripts.")
try:
script_config.write_config()
except zeek_exceptions.WriteZeekConfigError:
self.logger.error('Could not write Zeek script configuration.')
raise zeek_exceptions.InstallZeekError("Could not write Zeek script configuration.")
def setup_logstash_synesis(self):
"""
Create required environmental variables; copy configurations to various directories.
"""
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info(
'Creating Synesis installation and configuration directories.')
utilities.makedirs(self.install_directory, exist_ok=True)
self.logger.info('Copying Synesis configurations.')
utilities.copytree(
os.path.join(const.DEFAULT_CONFIGS, 'logstash', 'suricata'),
self.install_directory)
utilities.set_ownership_of_file(self.install_directory,
user='******',
group='dynamite')
try:
with open(env_file) as env_f:
env_str = env_f.read()
if 'SYNLITE_SURICATA_DICT_PATH' not in env_str:
dict_path = os.path.join(self.install_directory,
'dictionaries')
self.logger.info(
'Updating Synesis dictionary configuration path [{}]'.
format(dict_path))
subprocess.call(
'echo SYNLITE_SURICATA_DICT_PATH="{}" >> {}'.format(
dict_path, env_file),
shell=True)
if 'SYNLITE_SURICATA_TEMPLATE_PATH' not in env_str:
template_path = os.path.join(self.install_directory,
'templates')
self.logger.info(
'Updating Synesis template configuration path [{}]'.
format(template_path))
subprocess.call(
'echo SYNLITE_SURICATA_TEMPLATE_PATH="{}" >> {}'.
format(template_path, env_file),
shell=True)
if 'SYNLITE_SURICATA_GEOIP_DB_PATH' not in env_str:
geo_path = os.path.join(self.install_directory, 'geoipdbs')
self.logger.info(
'Updating Synesis GeoDBs configuration path [{}]'.
format(geo_path))
subprocess.call(
'echo SYNLITE_SURICATA_GEOIP_DB_PATH="{}" >> {}'.
format(geo_path, env_file),
shell=True)
except Exception as e:
self.logger.error('Failed to read Synesis environment variables.')
self.logger.debug(
"Failed to read Synesis environment variables; {}".format(e))
raise synesis_exceptions.InstallSynesisError(
"Failed to read Synesis environment variables; {} ".format(e))
try:
synesis_config.ConfigManager().write_environment_variables()
except (synesis_exceptions.ReadSynesisConfigError,
synesis_exceptions.WriteSynesisConfigError):
self.logger.error(
'Failed to read/write Synesis environment variables.')
raise synesis_exceptions.InstallSynesisError(
"Could not read/write Synesis environment variables.")
