def install(alsi=None):
alsi = alsi or SiteInstaller()
alsi.milestone("Install heuristics statistics...")
alsi.sudo_install_file('assemblyline/al/install/etc/cron/al-heuristics',
'/etc/cron.d/al-heuristics')
alsi.milestone("Completed installation of heuristic statistics.")
def install(alsi=None):
alsi = alsi or SiteInstaller()
if alsi.config.get('monitoring', {}).get('harddrive', False):
alsi.milestone("Install harddrive monitor...")
alsi.sudo_install_file(
'assemblyline/al/install/etc/cron/al-harddrive_monitor',
'/etc/cron.d/al-harddrive_monitor')
alsi.milestone("Completed installation of harddrive monitor.")
def install(alsi=None):
if not alsi:
from assemblyline.al.install import SiteInstaller
alsi = SiteInstaller()
alsi.pip_install_all(["elasticsearch==2.3.0"])
alsi.sudo_install_file('assemblyline/al/install/etc/init/metricsd.conf',
'/etc/init/metricsd.conf')
if not os.path.exists('/etc/init.d/metricsd'):
alsi.runcmd('sudo ln -s /lib/init/upstart-job /etc/init.d/metricsd')
def install(alsi=None):
if not alsi:
from assemblyline.al.install import SiteInstaller
alsi = SiteInstaller()
# Upstart Install.
alsi.sudo_install_file('assemblyline/al/install/etc/init/dispatcher_instance.conf',
'/etc/init/dispatcher_instance.conf')
alsi.sudo_install_file('assemblyline/al/install/etc/init/dispatchers.conf', '/etc/init/dispatchers.conf')
if not os.path.exists('/etc/init.d/dispatchers'):
alsi.runcmd('sudo ln -s /lib/init/upstart-job /etc/init.d/dispatchers')
def install(alsi=None):
if not alsi:
from assemblyline.al.install import SiteInstaller
alsi = SiteInstaller()
alsi.sudo_install_file('assemblyline/al/install/etc/init/plumber.conf',
'/etc/init/plumber.conf')
if not os.path.exists('/etc/init.d/plumber'):
alsi.runcmd('sudo ln -s /lib/init/upstart-job /etc/init.d/plumber')
def install(alsi=None):
if not alsi:
from assemblyline.al.install import SiteInstaller
alsi = SiteInstaller()
alsi.sudo_install_file('assemblyline/al/install/etc/init/controller.conf',
'/etc/init/controller.conf')
if not os.path.exists('/etc/init.d/controller'):
alsi.runcmd('sudo ln -s /lib/init/upstart-job /etc/init.d/controller')
tmpl_path = os.path.join(
alsi.alroot, 'pkg',
'assemblyline/al/install/etc/sudoers.d/controller.tmpl')
tmpl = open(tmpl_path).read()
alsi.append_line_if_doesnt_exist(
"/etc/sudoers", tmpl.replace('__USER__',
alsi.config['system']['user']))
def run_hooks_if_necessary(bstrap_cfg):
hooks = bstrap_cfg.get('installation', {}).get('hooks', {}).get('bootstrap', [])
if not hooks:
return
import sys
sys.path.append(AL_PYTHONPATH)
for runhook in hooks:
hook_module = importlib.import_module(runhook)
if not hasattr(hook_module, 'execute'):
logging.warn('hook: %s has no execute', hook_module)
return
from assemblyline.al.install import SiteInstaller
alsi = SiteInstaller(bstrap_cfg)
hook_cb = getattr(hook_module, 'execute')
hook_cb(alsi)
def install(alsi=None):
alsi = alsi or SiteInstaller()
alsi.milestone("Installing pureftp and default AL FTP config.")
alsi.sudo_apt_install(['pure-ftpd'])
user = alsi.config['filestore']['ftp_user']
sys_user = alsi.config['system']['user']
password = alsi.config['filestore']['ftp_password']
root = alsi.config['filestore']['ftp_root']
ip_restriction = alsi.config['filestore'].get('ftp_ip_restriction', None)
if ip_restriction:
alsi.runcmd("( echo '{password}' ; echo '{password}') | "
"sudo pure-pw useradd {user} -r {ip_restriction} "
"-u {sys_user} -g adm -d {root}".format(
user=user,
sys_user=sys_user,
password=password,
ip_restriction=ip_restriction,
root=root),
raise_on_error=False)
else:
alsi.runcmd(
"( echo '{password}' ; echo '{password}') | "
"sudo pure-pw useradd {user} -u {sys_user} -g adm -d {root}".
format(user=user, sys_user=sys_user, password=password, root=root),
raise_on_error=False)
alsi.runcmd("sudo pure-pw mkdb", raise_on_error=False)
alsi.runcmd(
"sudo ln -s /etc/pure-ftpd/conf/PureDB /etc/pure-ftpd/auth/50PureDB",
raise_on_error=False)
alsi.sudo_install_file(
'assemblyline/al/install/etc/pure-ftpd/conf/MaxClientsNumber',
'/etc/pure-ftpd/conf/MaxClientsNumber')
alsi.runcmd('sudo service pure-ftpd restart')
def install(alsi=None):
import getpass
alsi = alsi or SiteInstaller()
user = alsi.config['system']['user']
password = alsi.config['system'].get('password', None)
root = alsi.config['system']['root']
alsi.info("Creating user: [{user}]".format(user=user))
if password:
# noinspection PyUnresolvedReferences
import crypt
crypt_pass = crypt.crypt(password, "%s_%s_%s" % (user, root, password))
alsi.runcmd('sudo useradd -d {root} -p {crypt_pass} {user}'.format(
user=user, root=root, crypt_pass=crypt_pass),
raise_on_error=False)
else:
alsi.runcmd('sudo useradd -d {root} {user}'.format(user=user,
root=root),
raise_on_error=False)
alsi.info("Making sure current user can write into the install directory")
alsi.runcmd("sudo chown -R {user}:adm {root}".format(
user=getpass.getuser(), root=root))
alsi.info("Completed cleanup.")
# Find the text between "origin" and "(fetch)"
url = re.search("origin\t([^ ]*) \\(fetch\\)", rem_stdout).group(1)
except AttributeError:
print "Could not find origin fetch url in"
print rem_stdout
sys.exit(0)
# Replace the last instance of "assemblyline" with "{repo}"
url = re.sub("(.*/)assemblyline(.*?)", r"\1{repo}\2", url)
try:
# Find the first line that start with "* " and save everything after that
branch = re.search("^\\* (.*)", br_stdout, re.MULTILINE).group(1)
except AttributeError:
print "Could not find current branch in"
print br_stdout
sys.exit(0)
git_override = {'url': url, 'branch': branch}
except Exception, e:
git_override = None
args = sys.argv[1:]
if not args:
seed = 'assemblyline.al.install.seeds.assemblyline_common.DEFAULT_SEED'
else:
seed = args[0]
ssi = SiteInstaller(seed=seed, simple=True)
ssi.setup_git_repos(git_override=git_override)
#!/usr/bin/env python
from assemblyline.al.install import SiteInstaller
from assemblyline.al.install.stages import cmd_service_all
alsi = SiteInstaller()
alsi.milestone("Stoping components")
cmd_service_all(alsi, 'stop')
alsi.milestone("Installing pip configuration files")
alsi.install_persistent_pip_conf()
alsi.milestone("Creating AL user")
from assemblyline.al.install.stages import install_00_init
install_00_init.install(alsi)
alsi.milestone("Installing Bootstrap")
from assemblyline.al.install.stages import install_20_bootstrap
install_20_bootstrap.install(alsi)
alsi.milestone("Installing Common Dependencies")
from assemblyline.al.install.stages import install_30_core_deps
install_30_core_deps.install(alsi)
alsi.milestone("Patching /etc/hosts to route datastore.al to localhost")
from assemblyline.al.install.stages import install_40_patch_hosts
install_40_patch_hosts.install(alsi)
alsi.milestone("Installing Harddrive monitor")
from assemblyline.al.install.stages import install_40_harddrive_monitor
pypi_index_url=temp_config['installation']['pip_index_url'])
pip_installer.upgrade_all(['requests==2.10.0'])
# noinspection PyBroadException
try:
pip_installer.upgrade_all(['pip==8.1.2'])
except:
pass
else:
print "Cannot pip installer without the AL_SEED variable set"
exit(1)
# Start SiteInstaller
from assemblyline.al.install import SiteInstaller
alsi = SiteInstaller()
support_dir = os.path.join(alsi.alroot, 'support')
if not os.path.exists(support_dir):
os.makedirs(support_dir)
# Install boostrap packages
try:
# noinspection PyUnresolvedReferences
import Crypto
except ImportError:
pycrypto = 'pycrypto-2.6.win32-py2.7.exe'
local_path = os.path.join(support_dir, pycrypto)
alsi.fetch_package(r'python/pywin/' + pycrypto, local_path)
alsi.runcmd(local_path)
#!/usr/bin/env python
import os
def install(alsi):
alsi.sudo_apt_install('libpq-dev')
alsi.pip_install('psycopg2')
if __name__ == '__main__':
from assemblyline.al.install import SiteInstaller
alsi = SiteInstaller()
install(alsi)
def exec_on_all(cmd):
alsi = SiteInstaller()
cmd_service_all(alsi, cmd, al_svc_only=True)
#!/usr/bin/env python
import os
def install(alsi):
alsi.sudo_install_file(
'assemblyline/al/install/etc/init/alert_actions.conf',
'/etc/init/alert_actions.conf')
if not os.path.exists('/etc/init.d/alert_actions'):
alsi.runcmd(
'sudo ln -s /lib/init/upstart-job /etc/init.d/alert_actions')
if __name__ == '__main__':
from assemblyline.al.install import SiteInstaller
installer = SiteInstaller()
install(installer)
#!/usr/bin/env python
import sys
import importlib
import os
from assemblyline.al.install import SiteInstaller
from assemblyline.al.install.stages import cmd_service_all
alsi = SiteInstaller()
if len(sys.argv) == 1:
alsi.error("No service specified")
exit(1)
service_list = []
for service in sys.argv[1:]:
if service not in alsi.config['services']['master_list']:
alsi.warn("Cannot find service '%s' in master service list. Service will be skipped ..." % service)
else:
service_list.append(service)
if not service_list:
alsi.error("No service remaining to process")
exit(1)
alsi.milestone("Stoping components")
cmd_service_all(alsi, 'stop')
#!/usr/bin/env python
from assemblyline.al.install import SiteInstaller
from assemblyline.al.install.stages import cmd_service_all
alsi = SiteInstaller()
alsi.milestone("Stoping components")
cmd_service_all(alsi, 'stop')
alsi.milestone("Installing pip configuration files")
alsi.install_persistent_pip_conf()
alsi.milestone("Creating AL user")
from assemblyline.al.install.stages import install_00_init
install_00_init.install(alsi)
alsi.milestone("Installing Bootstrap")
from assemblyline.al.install.stages import install_20_bootstrap
install_20_bootstrap.install(alsi)
alsi.milestone("Fix default AL exports for VM based systems")
alsi.sudo_sed_inline("/etc/default/al", [
"s/{installer_host}/datastore.al/".format(
installer_host=alsi.config['core']['nodes'][0])
])
alsi.milestone("Installing Core Dependancies")
from assemblyline.al.install.stages import install_30_core_deps
install_30_core_deps.install(alsi)
def install(alsi=None):
if not alsi:
from assemblyline.al.install import SiteInstaller
alsi = SiteInstaller()
alsi.sudo_install_file(
'assemblyline/al/install/etc/sysctl.d/10-dispatcher-overcommitmem.conf',
'/etc/sysctl.d/10-dispatcher-overcommitmem.conf')
alsi.runcmd('sudo sysctl vm.overcommit_memory=1')
alsi.sudo_apt_install(['redis-server'])
alsi.sudo_sed_inline('/etc/redis/redis.conf', [
's/bind 127.0.0.1/bind 0.0.0.0/g', 's/timeout 0/timeout 30/g',
's/save 900 1/#save 900 1/g', 's/save 300 10/#save 300 10/g',
's/save 60 10000/save ""/g'
])
redis_cfg_tmp = '/tmp/redis.conf'
if os.path.exists(redis_cfg_tmp):
os.unlink(redis_cfg_tmp)
shutil.copyfile('/etc/redis/redis.conf', redis_cfg_tmp)
alsi.sudo_sed_inline(redis_cfg_tmp, [
's/redis-server.pid/redis-persist.pid/g', 's/port 6379/port 6380/g',
's/redis-server.log/redis-persist.log/g',
's/# syslog-ident redis/syslog-ident redis-persist/g',
's|/var/lib/redis|/var/lib/redis-persist|g',
's/appendonly no/appendonly yes/g',
's/auto-aof-rewrite-percentage 100/auto-aof-rewrite-percentage 0/g'
])
alsi.sudo_sed_inline('/etc/default/redis-server',
['s/# ULIMIT=/ULIMIT=/g'])
alsi.sudo_install_file(redis_cfg_tmp, '/etc/redis-persist/redis.conf')
redis_init_tmp = '/tmp/init_redis.conf'
if os.path.exists(redis_init_tmp):
os.unlink(redis_init_tmp)
shutil.copyfile('/etc/init.d/redis-server', redis_init_tmp)
alsi.sudo_sed_inline(redis_init_tmp, [
r's/redis\/redis.conf/redis-persist\/redis.conf/g',
's/redis-server.pid/redis-persist.pid/g'
])
alsi.sudo_install_file(redis_init_tmp, '/etc/init.d/redis-persist')
alsi.runcmd('sudo mkdir /var/lib/redis-persist', raise_on_error=False)
alsi.runcmd('sudo chown redis:redis /var/lib/redis-persist')
alsi.runcmd('sudo update-rc.d redis-persist defaults')
persistent_settings = alsi.config['core']['redis']['persistent']
db = persistent_settings['db']
port = persistent_settings['port']
tmpl_path = os.path.join(
alsi.alroot, 'pkg',
'assemblyline/al/install/etc/cron/al-redis_maintenance.tmpl')
tmpl = open(tmpl_path).read()
cfg = tmpl.replace('___DB___', str(db)).replace('___PORT___', str(port))
with open('/tmp/al-redis_maintenance', 'w') as f:
f.write(cfg)
alsi.sudo_install_file('/tmp/al-redis_maintenance',
'/etc/cron.d/al-redis_maintenance')
#!/usr/bin/env python
from assemblyline.al.install.seeds.assemblyline_common import seed
from assemblyline.al.install import SiteInstaller
appliance_ip = SiteInstaller.get_ipaddress(silent=True)
SYS_PASS = '******'
SYS_USER = '******'
FTP_PASS = '******'
FTP_USER = '******'
# Start with the default seed and update for AL
seed['core']['alerter']['shards'] = 1
seed['core']['dispatcher']['shards'] = 1
seed['core']['expiry']['delete_storage'] = False
seed['core']['middleman']['shards'] = 1
seed['core']['nodes'] = [appliance_ip]
seed['core']['redis']['persistent']['host'] = appliance_ip
seed['core']['redis']['nonpersistent']['host'] = appliance_ip
seed['datastore']['port'] = 9087
seed['datastore']['stream_port'] = 9098
seed['datastore']['solr_port'] = 9093
seed['datastore']['riak']['solr']['heap_max_gb'] = 2
seed['datastore']['riak']['nodes'] = [appliance_ip]
seed['datastore']['riak']['ring_size'] = 32
seed['datastore']['riak']['nvals'] = {'low': 1, 'med': 1, 'high': 1}
seed['filestore']['ftp_password'] = FTP_PASS
#!/usr/bin/env python
import os
import zipfile
def install(alsi):
support_dir = os.path.join(alsi.alroot, 'support')
if not os.path.exists(support_dir):
os.makedirs(support_dir)
alsi.install_pefile()
local_sigcheck_zip = os.path.join(support_dir, 'sigcheck.zip')
alsi.fetch_package("sigcheck/sigcheck.zip", local_sigcheck_zip)
with zipfile.ZipFile(local_sigcheck_zip) as zf:
zf.extractall(path=support_dir)
# TODO: Update certificate catalogue ??
if __name__ == '__main__':
from assemblyline.al.install import SiteInstaller
install(SiteInstaller())
#!/usr/bin/env python
from assemblyline.al.install import SiteInstaller
from assemblyline.al.install.stages import cmd_service_all
alsi = SiteInstaller()
alsi.milestone("Stoping components")
cmd_service_all(alsi, 'stop')
alsi.milestone("Installing pip configuration files")
alsi.install_persistent_pip_conf()
alsi.milestone("Creating AL user")
from assemblyline.al.install.stages import install_00_init
install_00_init.install(alsi)
alsi.milestone("Setup and clone git repos")
alsi.setup_git_repos()
alsi.milestone("Reload configuration")
alsi.reload_config()
alsi.milestone("Installing Bootstrap")
from assemblyline.al.install.stages import install_20_bootstrap
install_20_bootstrap.install(alsi)
alsi.milestone("Installing Core Dependancies")
from assemblyline.al.install.stages import install_30_core_deps
install_30_core_deps.install(alsi)