def install(alsi=None):
    alsi = alsi or SiteInstaller()

    alsi.milestone("Install heuristics statistics...")
    alsi.sudo_install_file('assemblyline/al/install/etc/cron/al-heuristics',
                           '/etc/cron.d/al-heuristics')
    alsi.milestone("Completed installation of heuristic statistics.")
示例#2
0
def install(alsi=None):
    alsi = alsi or SiteInstaller()

    if alsi.config.get('monitoring', {}).get('harddrive', False):
        alsi.milestone("Install harddrive monitor...")
        alsi.sudo_install_file(
            'assemblyline/al/install/etc/cron/al-harddrive_monitor',
            '/etc/cron.d/al-harddrive_monitor')
        alsi.milestone("Completed installation of harddrive monitor.")
def install(alsi=None):
    if not alsi:
        from assemblyline.al.install import SiteInstaller
        alsi = SiteInstaller()

    alsi.pip_install_all(["elasticsearch==2.3.0"])

    alsi.sudo_install_file('assemblyline/al/install/etc/init/metricsd.conf',
                           '/etc/init/metricsd.conf')

    if not os.path.exists('/etc/init.d/metricsd'):
        alsi.runcmd('sudo ln -s /lib/init/upstart-job /etc/init.d/metricsd')
def install(alsi=None):

    if not alsi:
        from assemblyline.al.install import SiteInstaller
        alsi = SiteInstaller()

    # Upstart Install.
    alsi.sudo_install_file('assemblyline/al/install/etc/init/dispatcher_instance.conf',
                          '/etc/init/dispatcher_instance.conf')

    alsi.sudo_install_file('assemblyline/al/install/etc/init/dispatchers.conf', '/etc/init/dispatchers.conf')

    if not os.path.exists('/etc/init.d/dispatchers'):
        alsi.runcmd('sudo ln -s /lib/init/upstart-job /etc/init.d/dispatchers')
示例#5
0
def install(alsi=None):
    if not alsi:
        from assemblyline.al.install import SiteInstaller
        alsi = SiteInstaller()

    alsi.sudo_install_file('assemblyline/al/install/etc/init/plumber.conf',
                           '/etc/init/plumber.conf')

    if not os.path.exists('/etc/init.d/plumber'):
        alsi.runcmd('sudo ln -s /lib/init/upstart-job /etc/init.d/plumber')
def install(alsi=None):
    if not alsi:
        from assemblyline.al.install import SiteInstaller
        alsi = SiteInstaller()

    alsi.sudo_install_file('assemblyline/al/install/etc/init/controller.conf',
                           '/etc/init/controller.conf')

    if not os.path.exists('/etc/init.d/controller'):
        alsi.runcmd('sudo ln -s /lib/init/upstart-job /etc/init.d/controller')

    tmpl_path = os.path.join(
        alsi.alroot, 'pkg',
        'assemblyline/al/install/etc/sudoers.d/controller.tmpl')
    tmpl = open(tmpl_path).read()
    alsi.append_line_if_doesnt_exist(
        "/etc/sudoers", tmpl.replace('__USER__',
                                     alsi.config['system']['user']))
示例#7
0
def run_hooks_if_necessary(bstrap_cfg):
    hooks = bstrap_cfg.get('installation', {}).get('hooks', {}).get('bootstrap', [])
    if not hooks:
        return

    import sys
    sys.path.append(AL_PYTHONPATH)

    for runhook in hooks:
        hook_module = importlib.import_module(runhook)
        if not hasattr(hook_module, 'execute'):
            logging.warn('hook: %s has no execute', hook_module)
            return

        from assemblyline.al.install import SiteInstaller
        alsi = SiteInstaller(bstrap_cfg)
        hook_cb = getattr(hook_module, 'execute')
        hook_cb(alsi)
示例#8
0
def install(alsi=None):
    alsi = alsi or SiteInstaller()

    alsi.milestone("Installing pureftp and default AL FTP config.")
    alsi.sudo_apt_install(['pure-ftpd'])

    user = alsi.config['filestore']['ftp_user']
    sys_user = alsi.config['system']['user']
    password = alsi.config['filestore']['ftp_password']
    root = alsi.config['filestore']['ftp_root']
    ip_restriction = alsi.config['filestore'].get('ftp_ip_restriction', None)

    if ip_restriction:
        alsi.runcmd("( echo '{password}' ; echo '{password}') | "
                    "sudo pure-pw useradd {user} -r {ip_restriction} "
                    "-u {sys_user} -g adm -d {root}".format(
                        user=user,
                        sys_user=sys_user,
                        password=password,
                        ip_restriction=ip_restriction,
                        root=root),
                    raise_on_error=False)
    else:
        alsi.runcmd(
            "( echo '{password}' ; echo '{password}') | "
            "sudo pure-pw useradd {user} -u {sys_user} -g adm -d {root}".
            format(user=user, sys_user=sys_user, password=password, root=root),
            raise_on_error=False)

    alsi.runcmd("sudo pure-pw mkdb", raise_on_error=False)
    alsi.runcmd(
        "sudo ln -s /etc/pure-ftpd/conf/PureDB /etc/pure-ftpd/auth/50PureDB",
        raise_on_error=False)

    alsi.sudo_install_file(
        'assemblyline/al/install/etc/pure-ftpd/conf/MaxClientsNumber',
        '/etc/pure-ftpd/conf/MaxClientsNumber')
    alsi.runcmd('sudo service pure-ftpd restart')
示例#9
0
def install(alsi=None):
    import getpass
    alsi = alsi or SiteInstaller()
    user = alsi.config['system']['user']
    password = alsi.config['system'].get('password', None)
    root = alsi.config['system']['root']
    alsi.info("Creating user: [{user}]".format(user=user))
    if password:
        # noinspection PyUnresolvedReferences
        import crypt
        crypt_pass = crypt.crypt(password, "%s_%s_%s" % (user, root, password))
        alsi.runcmd('sudo useradd -d {root} -p {crypt_pass} {user}'.format(
            user=user, root=root, crypt_pass=crypt_pass),
                    raise_on_error=False)
    else:
        alsi.runcmd('sudo useradd -d {root} {user}'.format(user=user,
                                                           root=root),
                    raise_on_error=False)

    alsi.info("Making sure current user can write into the install directory")
    alsi.runcmd("sudo chown -R {user}:adm {root}".format(
        user=getpass.getuser(), root=root))
    alsi.info("Completed cleanup.")
示例#10
0
            # Find the text between "origin" and "(fetch)"
            url = re.search("origin\t([^ ]*) \\(fetch\\)", rem_stdout).group(1)
        except AttributeError:
            print "Could not find origin fetch url in"
            print rem_stdout
            sys.exit(0)

        # Replace the last instance of "assemblyline" with "{repo}"
        url = re.sub("(.*/)assemblyline(.*?)", r"\1{repo}\2", url)

        try:
            # Find the first line that start with "* " and save everything after that
            branch = re.search("^\\* (.*)", br_stdout, re.MULTILINE).group(1)
        except AttributeError:
            print "Could not find current branch in"
            print br_stdout
            sys.exit(0)

        git_override = {'url': url, 'branch': branch}
    except Exception, e:
        git_override = None

    args = sys.argv[1:]
    if not args:
        seed = 'assemblyline.al.install.seeds.assemblyline_common.DEFAULT_SEED'
    else:
        seed = args[0]

    ssi = SiteInstaller(seed=seed, simple=True)
    ssi.setup_git_repos(git_override=git_override)
示例#11
0
#!/usr/bin/env python

from assemblyline.al.install import SiteInstaller
from assemblyline.al.install.stages import cmd_service_all

alsi = SiteInstaller()

alsi.milestone("Stoping components")
cmd_service_all(alsi, 'stop')

alsi.milestone("Installing pip configuration files")
alsi.install_persistent_pip_conf()

alsi.milestone("Creating AL user")
from assemblyline.al.install.stages import install_00_init
install_00_init.install(alsi)

alsi.milestone("Installing Bootstrap")
from assemblyline.al.install.stages import install_20_bootstrap
install_20_bootstrap.install(alsi)

alsi.milestone("Installing Common Dependencies")
from assemblyline.al.install.stages import install_30_core_deps
install_30_core_deps.install(alsi)

alsi.milestone("Patching /etc/hosts to route datastore.al to localhost")
from assemblyline.al.install.stages import install_40_patch_hosts
install_40_patch_hosts.install(alsi)

alsi.milestone("Installing Harddrive monitor")
from assemblyline.al.install.stages import install_40_harddrive_monitor
示例#12
0
            pypi_index_url=temp_config['installation']['pip_index_url'])
        pip_installer.upgrade_all(['requests==2.10.0'])
        # noinspection PyBroadException
        try:
            pip_installer.upgrade_all(['pip==8.1.2'])
        except:
            pass

    else:
        print "Cannot pip installer without the AL_SEED variable set"
        exit(1)

# Start SiteInstaller
from assemblyline.al.install import SiteInstaller

alsi = SiteInstaller()

support_dir = os.path.join(alsi.alroot, 'support')
if not os.path.exists(support_dir):
    os.makedirs(support_dir)

# Install boostrap packages
try:
    # noinspection PyUnresolvedReferences
    import Crypto
except ImportError:
    pycrypto = 'pycrypto-2.6.win32-py2.7.exe'
    local_path = os.path.join(support_dir, pycrypto)
    alsi.fetch_package(r'python/pywin/' + pycrypto, local_path)
    alsi.runcmd(local_path)
示例#13
0
#!/usr/bin/env python

import os


def install(alsi):
    alsi.sudo_apt_install('libpq-dev')
    alsi.pip_install('psycopg2')


if __name__ == '__main__':
    from assemblyline.al.install import SiteInstaller
    alsi = SiteInstaller()
    install(alsi)
示例#14
0
def exec_on_all(cmd):
    alsi = SiteInstaller()
    cmd_service_all(alsi, cmd, al_svc_only=True)
示例#15
0
#!/usr/bin/env python
import os


def install(alsi):
    alsi.sudo_install_file(
        'assemblyline/al/install/etc/init/alert_actions.conf',
        '/etc/init/alert_actions.conf')

    if not os.path.exists('/etc/init.d/alert_actions'):
        alsi.runcmd(
            'sudo ln -s /lib/init/upstart-job /etc/init.d/alert_actions')


if __name__ == '__main__':
    from assemblyline.al.install import SiteInstaller
    installer = SiteInstaller()
    install(installer)
示例#16
0
#!/usr/bin/env python

import sys
import importlib
import os

from assemblyline.al.install import SiteInstaller
from assemblyline.al.install.stages import cmd_service_all

alsi = SiteInstaller()


if len(sys.argv) == 1:
    alsi.error("No service specified")
    exit(1)

service_list = []

for service in sys.argv[1:]:
    if service not in alsi.config['services']['master_list']:
        alsi.warn("Cannot find service '%s' in master service list. Service will be skipped ..." % service)
    else:
        service_list.append(service)

if not service_list:
    alsi.error("No service remaining to process")
    exit(1)

alsi.milestone("Stoping components")
cmd_service_all(alsi, 'stop')
示例#17
0
#!/usr/bin/env python

from assemblyline.al.install import SiteInstaller
from assemblyline.al.install.stages import cmd_service_all

alsi = SiteInstaller()

alsi.milestone("Stoping components")
cmd_service_all(alsi, 'stop')

alsi.milestone("Installing pip configuration files")
alsi.install_persistent_pip_conf()

alsi.milestone("Creating AL user")
from assemblyline.al.install.stages import install_00_init
install_00_init.install(alsi)

alsi.milestone("Installing Bootstrap")
from assemblyline.al.install.stages import install_20_bootstrap
install_20_bootstrap.install(alsi)

alsi.milestone("Fix default AL exports for VM based systems")
alsi.sudo_sed_inline("/etc/default/al", [
    "s/{installer_host}/datastore.al/".format(
        installer_host=alsi.config['core']['nodes'][0])
])

alsi.milestone("Installing Core Dependancies")
from assemblyline.al.install.stages import install_30_core_deps
install_30_core_deps.install(alsi)
示例#18
0
def install(alsi=None):

    if not alsi:
        from assemblyline.al.install import SiteInstaller
        alsi = SiteInstaller()

    alsi.sudo_install_file(
        'assemblyline/al/install/etc/sysctl.d/10-dispatcher-overcommitmem.conf',
        '/etc/sysctl.d/10-dispatcher-overcommitmem.conf')
    alsi.runcmd('sudo sysctl vm.overcommit_memory=1')

    alsi.sudo_apt_install(['redis-server'])

    alsi.sudo_sed_inline('/etc/redis/redis.conf', [
        's/bind 127.0.0.1/bind 0.0.0.0/g', 's/timeout 0/timeout 30/g',
        's/save 900 1/#save 900 1/g', 's/save 300 10/#save 300 10/g',
        's/save 60 10000/save ""/g'
    ])

    redis_cfg_tmp = '/tmp/redis.conf'
    if os.path.exists(redis_cfg_tmp):
        os.unlink(redis_cfg_tmp)
    shutil.copyfile('/etc/redis/redis.conf', redis_cfg_tmp)

    alsi.sudo_sed_inline(redis_cfg_tmp, [
        's/redis-server.pid/redis-persist.pid/g', 's/port 6379/port 6380/g',
        's/redis-server.log/redis-persist.log/g',
        's/# syslog-ident redis/syslog-ident redis-persist/g',
        's|/var/lib/redis|/var/lib/redis-persist|g',
        's/appendonly no/appendonly yes/g',
        's/auto-aof-rewrite-percentage 100/auto-aof-rewrite-percentage 0/g'
    ])

    alsi.sudo_sed_inline('/etc/default/redis-server',
                         ['s/# ULIMIT=/ULIMIT=/g'])

    alsi.sudo_install_file(redis_cfg_tmp, '/etc/redis-persist/redis.conf')

    redis_init_tmp = '/tmp/init_redis.conf'
    if os.path.exists(redis_init_tmp):
        os.unlink(redis_init_tmp)
    shutil.copyfile('/etc/init.d/redis-server', redis_init_tmp)

    alsi.sudo_sed_inline(redis_init_tmp, [
        r's/redis\/redis.conf/redis-persist\/redis.conf/g',
        's/redis-server.pid/redis-persist.pid/g'
    ])
    alsi.sudo_install_file(redis_init_tmp, '/etc/init.d/redis-persist')

    alsi.runcmd('sudo mkdir /var/lib/redis-persist', raise_on_error=False)
    alsi.runcmd('sudo chown redis:redis /var/lib/redis-persist')
    alsi.runcmd('sudo update-rc.d redis-persist defaults')

    persistent_settings = alsi.config['core']['redis']['persistent']
    db = persistent_settings['db']
    port = persistent_settings['port']

    tmpl_path = os.path.join(
        alsi.alroot, 'pkg',
        'assemblyline/al/install/etc/cron/al-redis_maintenance.tmpl')

    tmpl = open(tmpl_path).read()
    cfg = tmpl.replace('___DB___', str(db)).replace('___PORT___', str(port))
    with open('/tmp/al-redis_maintenance', 'w') as f:
        f.write(cfg)

    alsi.sudo_install_file('/tmp/al-redis_maintenance',
                           '/etc/cron.d/al-redis_maintenance')
#!/usr/bin/env python

from assemblyline.al.install.seeds.assemblyline_common import seed
from assemblyline.al.install import SiteInstaller

appliance_ip = SiteInstaller.get_ipaddress(silent=True)

SYS_PASS = '******'
SYS_USER = '******'
FTP_PASS = '******'
FTP_USER = '******'

# Start with the default seed and update for AL
seed['core']['alerter']['shards'] = 1
seed['core']['dispatcher']['shards'] = 1
seed['core']['expiry']['delete_storage'] = False
seed['core']['middleman']['shards'] = 1
seed['core']['nodes'] = [appliance_ip]
seed['core']['redis']['persistent']['host'] = appliance_ip
seed['core']['redis']['nonpersistent']['host'] = appliance_ip

seed['datastore']['port'] = 9087
seed['datastore']['stream_port'] = 9098
seed['datastore']['solr_port'] = 9093

seed['datastore']['riak']['solr']['heap_max_gb'] = 2
seed['datastore']['riak']['nodes'] = [appliance_ip]
seed['datastore']['riak']['ring_size'] = 32
seed['datastore']['riak']['nvals'] = {'low': 1, 'med': 1, 'high': 1}

seed['filestore']['ftp_password'] = FTP_PASS
示例#20
0
#!/usr/bin/env python

import os
import zipfile


def install(alsi):
    support_dir = os.path.join(alsi.alroot, 'support')
    if not os.path.exists(support_dir):
        os.makedirs(support_dir)

    alsi.install_pefile()
    local_sigcheck_zip = os.path.join(support_dir, 'sigcheck.zip')
    alsi.fetch_package("sigcheck/sigcheck.zip", local_sigcheck_zip)
    with zipfile.ZipFile(local_sigcheck_zip) as zf:
        zf.extractall(path=support_dir)

    # TODO: Update certificate catalogue ??


if __name__ == '__main__':
    from assemblyline.al.install import SiteInstaller
    install(SiteInstaller())
示例#21
0
#!/usr/bin/env python

from assemblyline.al.install import SiteInstaller
from assemblyline.al.install.stages import cmd_service_all

alsi = SiteInstaller()

alsi.milestone("Stoping components")
cmd_service_all(alsi, 'stop')

alsi.milestone("Installing pip configuration files")
alsi.install_persistent_pip_conf()

alsi.milestone("Creating AL user")
from assemblyline.al.install.stages import install_00_init
install_00_init.install(alsi)

alsi.milestone("Setup and clone git repos")
alsi.setup_git_repos()

alsi.milestone("Reload configuration")
alsi.reload_config()

alsi.milestone("Installing Bootstrap")
from assemblyline.al.install.stages import install_20_bootstrap
install_20_bootstrap.install(alsi)

alsi.milestone("Installing Core Dependancies")
from assemblyline.al.install.stages import install_30_core_deps
install_30_core_deps.install(alsi)