Ejemplo n.º 1
0
def initialize_test_environment():
    global test_dir

    # there is no reason to run anything as root
    if os.geteuid() == 0:
        print("do not run ace as root please")
        sys.exit(1)

    # where is ACE?
    saq_home = '/opt/saq'
    if 'SAQ_HOME' in os.environ:
        saq_home = os.environ['SAQ_HOME']

    # adjust search path
    sys.path.append(os.path.join(saq_home, 'lib'))

    # initialize saq
    import saq
    saq.initialize(saq_home=saq_home, config_paths=[], 
                   logging_config_path=os.path.join(saq_home, 'etc', 'unittest_logging.ini'), 
                   args=None, relative_dir=None, unittest=True)

    if saq.CONFIG['global']['instance_type'] not in [ 'PRODUCTION', 'QA', 'DEV' ]:
        sys.stderr.write('\n\n *** CRITICAL ERROR *** \n\ninvalid instance_type setting in configuration\n')
        sys.exit(1)

    if saq.CONFIG['global']['instance_type'] == 'PRODUCTION':
        sys.stderr.write('\n\n *** PROTECT PRODUCTION *** \ndo not execute this in production, idiot\n')
        sys.exit(1)

    # additional logging required for testing
    initialize_unittest_logging()

    # create a temporary storage directory
    test_dir = os.path.join(saq.SAQ_HOME, 'var', 'test')
    if os.path.exists(test_dir):
        try:
            shutil.rmtree(test_dir)
        except Exception as e:
            logging.error("unable to delete {}: {}".format(test_dir, e))
            sys.exit(1)

    try:
        os.makedirs(test_dir)
    except Exception as e:
        logging.error("unable to create temp dir {}: {}".format(test_dir, e))

    # in all our testing we use the password "password" for encryption/decryption
    from saq.crypto import get_aes_key
    saq.ENCRYPTION_PASSWORD = get_aes_key('password')

    #initialize_database()
    initialized = True
Ejemplo n.º 2
0
def initialize_test_environment():
    global initialized
    global test_dir

    if initialized:
        return

    # there is no reason to run anything as root
    if os.geteuid() == 0:
        print("do not run ace as root please")
        sys.exit(1)

    # where is ACE?
    saq_home = '/opt/saq'
    if 'SAQ_HOME' in os.environ:
        saq_home = os.environ['SAQ_HOME']

    # adjust search path
    sys.path.append(os.path.join(saq_home, 'lib'))

    # initialize saq
    import saq
    saq.initialize(saq_home=saq_home,
                   config_paths=[],
                   logging_config_path=os.path.join(saq_home, 'etc',
                                                    'unittest_logging.ini'),
                   args=None,
                   relative_dir=None)

    # additional logging required for testing
    initialize_unittest_logging()

    # create a temporary storage directory
    test_dir = os.path.join(saq.SAQ_HOME, 'var', 'test')
    if os.path.exists(test_dir):
        try:
            shutil.rmtree(test_dir)
        except Exception as e:
            logging.error("unable to delete {}: {}".format(test_dir, e))
            sys.exit(1)

    try:
        os.makedirs(test_dir)
    except Exception as e:
        logging.error("unable to create temp dir {}: {}".format(test_dir, e))

    # in all our testing we use the password "password" for encryption/decryption
    from saq.crypto import get_aes_key
    saq.ENCRYPTION_PASSWORD = get_aes_key('password')

    initialize_database()
    initialized = True
Ejemplo n.º 3
0
    def reset(self):
        """Resets everything back to the default state."""
        self.reset_config()
        self.reset_hal9000()
        self.reset_brocess()
        self.reset_cloudphish()
        self.reset_correlation()
        self.reset_email_archive()
        self.reset_crawlphish()
        self.reset_log_exports()
        self.reset_var_dir()

        # re-enable encryption in case we disabled it
        saq.ENCRYPTION_PASSWORD = get_aes_key('password')
Ejemplo n.º 4
0
 def setUp(self, *args, **kwargs):
     super().setUp(*args, **kwargs)
     self.old_password = saq.ENCRYPTION_PASSWORD
     saq.ENCRYPTION_PASSWORD = get_aes_key('test')
Ejemplo n.º 5
0
def initialize(saq_home=None, 
               config_paths=None, 
               logging_config_path=None, 
               args=None, 
               relative_dir=None):

    from saq.database import initialize_database, initialize_node, initialize_automation_user

    global API_PREFIX
    global AUTOMATION_USER_ID
    global CA_CHAIN_PATH
    global COMPANY_ID
    global COMPANY_NAME
    global CONFIG
    global CONFIG_PATHS
    global DAEMON_DIR
    global DAEMON_MODE
    global DATA_DIR
    global DEFAULT_ENCODING
    global DUMP_TRACEBACKS
    global ECS_SOCKET_PATH
    global ENCRYPTION_INITIALIZED
    global ENCRYPTION_PASSWORD
    global ENCRYPTION_PASSWORD_PLAINTEXT
    global EXCLUDED_SLA_ALERT_TYPES
    global EXECUTION_THREAD_LONG_TIMEOUT
    global FORCED_ALERTS
    global GLOBAL_SLA_SETTINGS
    global GUI_WHITELIST_EXCLUDED_OBSERVABLE_TYPES
    global INSTANCE_TYPE
    global LOCK_TIMEOUT_SECONDS
    global LOG_DIRECTORY
    global LOG_LEVEL
    global MANAGED_NETWORKS
    global MODULE_STATS_DIR
    global OTHER_PROXIES 
    global OTHER_SLA_SETTINGS
    global PROXIES
    global SAQ_HOME
    global SAQ_NODE
    global SAQ_NODE_ID
    global SAQ_RELATIVE_DIR
    global SEMAPHORES_ENABLED
    global SERVICES_DIR
    global STATS_DIR
    global TEMP_DIR
    global TOR_PROXY

    SAQ_HOME = None
    SAQ_NODE = None
    SAQ_NODE_ID = None
    API_PREFIX = None
    SAQ_RELATIVE_DIR = None
    CONFIG = None
    CONFIG_PATHS = []
    DATA_DIR = None
    TEMP_DIR = None
    DEFAULT_ENCODING = None
    SEMAPHORES_ENABLED = False
    PROXIES = {}
    OTHER_PROXIES = {}
    TOR_PROXY = None
    # list of iptools.IpRange objects defined in [network_configuration]
    MANAGED_NETWORKS = None
    # set this to True to force all anlaysis to result in an alert being generated
    FORCED_ALERTS = False
    # the private key password for encrypting/decrypting archive files
    # NOTE this is the decrypted random string of bytes that is used to encrypt/decrypt using AES
    # NOTE both of these can stay None if encryption is not being used
    ENCRYPTION_PASSWORD = None
    # *this* is the password that is used to encrypt/decrypt the ENCRYPTION_PASSWORD at rest
    ENCRYPTION_PASSWORD_PLAINTEXT = None
    # set to True after we've initialized encryption
    ENCRYPTION_INITIALIZED = False

    # the global log level setting
    LOG_LEVEL = logging.INFO
    # global logging directory (relative to DATA_DIR)
    LOG_DIRECTORY = None

    # directory containing statistical runtime info
    STATS_DIR = None 
    MODULE_STATS_DIR = None

    # are we running as a daemon in the background?
    DAEMON_MODE = False

    # directory where pid files are stored for daemons
    DAEMON_DIR = None

    # directory where files are stored for running services
    SERVICES_DIR = None

    # path to the certifcate chain used by all SSL certs
    CA_CHAIN_PATH = None

    # what type of instance is this?
    INSTANCE_TYPE = INSTANCE_TYPE_PRODUCTION

    # SLA settings
    GLOBAL_SLA_SETTINGS = None
    OTHER_SLA_SETTINGS = []
    EXCLUDED_SLA_ALERT_TYPES = []

    # set to True to cause tracebacks to be dumped to standard output
    # useful when debugging or testing
    DUMP_TRACEBACKS = False

    # the amount of time (in seconds) that a lock in the locks table is valid
    LOCK_TIMEOUT_SECONDS = None

    # amount of time (in seconds) before a process blows up because a threaded module won't stop
    EXECUTION_THREAD_LONG_TIMEOUT = None

    # the company/custom this node belongs to
    COMPANY_NAME = None
    COMPANY_ID = None

    # go ahead and try to figure out what text encoding we're using
    DEFAULT_ENCODING = locale.getpreferredencoding()

    # list of observable types we want to exclude from whitelisting (via the GUI)
    GUI_WHITELIST_EXCLUDED_OBSERVABLE_TYPES = []

    # do we want to force alerts?
    if args:
        FORCED_ALERTS = args.force_alerts

    # what is the root directory of the entire system?
    if saq_home is not None:
        SAQ_HOME = saq_home
    elif 'SAQ_HOME' in os.environ:
        SAQ_HOME = os.environ['SAQ_HOME']
    else:
        SAQ_HOME = '.'

    if not os.path.isdir(SAQ_HOME):
        sys.stderr.write("invalid root SAQ directory {0}\n".format(SAQ_HOME)) 
        sys.exit(1)

    # path to the unix socket for the encryption cache service
    ECS_SOCKET_PATH = os.path.join(SAQ_HOME, '.ecs')

    # XXX not sure we need this SAQ_RELATIVE_DIR anymore -- check it out
    # this system was originally designed to run out of /opt/saq
    # later we modified to run out of anywhere for command line correlation
    # when running the GUI in apache you have no control over the current working directory
    # so we specify what directory we'd *want* to be running out of here (even if we're not actually)
    # this only matters when loading alerts
    # this defaults to the current working directory
    SAQ_RELATIVE_DIR = os.path.relpath(os.getcwd(), start=SAQ_HOME)
    if relative_dir:
        SAQ_RELATIVE_DIR = relative_dir

    # load configuration file
    # defaults to $SAQ_HOME/etc/saq.ini
    if args:
        if args.config_paths:
            config_paths = args.config_paths

    if config_paths is None:
        config_paths = []
    
    # make each relative config path absolute to SAQ_HOME
    CONFIG_PATHS = [os.path.join(SAQ_HOME, p) if not os.path.isabs(p) else p for p in config_paths]

    # add any config files specified in SAQ_CONFIG_PATHS env var (command separated)
    if 'SAQ_CONFIG_PATHS' in os.environ:
        for config_path in os.environ['SAQ_CONFIG_PATHS'].split(','):
            config_path = config_path.strip()
            if not os.path.isabs(config_path):
                config_path = os.path.join(SAQ_HOME, config_path)
            if not os.path.exists(config_path):
                sys.stderr.write("WARNING: config path {} specified in SAQ_CONFIG_PATHS env var does not exist\n".format(config_path))
            else:
                if config_path not in CONFIG_PATHS:
                    CONFIG_PATHS.append(config_path)

    if UNIT_TESTING:
        # unit testing loads different configurations
        CONFIG_PATHS.append(os.path.join(SAQ_HOME, 'etc', 'saq.unittest.default.ini'))
        CONFIG_PATHS.append(os.path.join(SAQ_HOME, 'etc', 'saq.unittest.ini'))
    else:
        CONFIG_PATHS.append(os.path.join(SAQ_HOME, 'etc', 'saq.ini'))

    try:
        load_configuration()
    except Exception as e:
        sys.stderr.write("ERROR: unable to load configuration: {0}".format(str(e)))
        sys.exit(1)

    DATA_DIR = os.path.join(SAQ_HOME, CONFIG['global']['data_dir'])
    TEMP_DIR = os.path.join(DATA_DIR, CONFIG['global']['tmp_dir'])
    DAEMON_DIR = os.path.join(DATA_DIR, 'var', 'daemon')
    SERVICES_DIR = os.path.join(DATA_DIR, 'var', 'services')
    COMPANY_NAME = CONFIG['global']['company_name']
    COMPANY_ID = CONFIG['global'].getint('company_id')

    minutes, seconds = map(int, CONFIG['global']['lock_timeout'].split(':'))
    LOCK_TIMEOUT_SECONDS = (minutes * 60) + seconds
    EXECUTION_THREAD_LONG_TIMEOUT = CONFIG['global'].getint('execution_thread_long_timeout')

    # user specified log level
    LOG_LEVEL = logging.INFO
    if args:
        if args.log_level:
            LOG_LEVEL = args.log_level

    # make sure the logs directory exists
    LOG_DIRECTORY = os.path.join(DATA_DIR, 'logs')
    if not os.path.exists(LOG_DIRECTORY):
        try:
            os.mkdir(LOG_DIRECTORY)
        except Exception as e:
            sys.stderr.write("unable to mkdir {}: {}\n".format(LOG_DIRECTORY, e))
            sys.exit(1)

    # by default we log to the console
    if logging_config_path is None:
        logging_config_path = os.path.join(SAQ_HOME, 'etc', 'console_logging.ini')

    # we can override this on the command line
    # this is what we use for production engine settings
    if args:
        if args.logging_config_path:
            logging_config_path = args.logging_config_path
    
    # we can re-initialize later if we have to
    try:
        initialize_logging(logging_config_path) # this log file just gets some startup information
    except Exception as e:
        sys.exit(1)

    # has the encryption password been set yet?
    import saq.crypto
    from saq.crypto import get_aes_key, InvalidPasswordError

    if not saq.UNIT_TESTING:
        # are we prompting for the decryption password?
        if args and args.provide_decryption_password:
            while True:
                ENCRYPTION_PASSWORD_PLAINTEXT = getpass("Enter the decryption password:"******"invalid encryption password")
                    continue

                break

        elif saq.crypto.encryption_key_set():
            # if we're not prompting for it, are we running the encryption cache service yet?
            logging.debug("reading encryption password from ecs")
            ENCRYPTION_PASSWORD_PLAINTEXT = saq.crypto.read_ecs()
            if ENCRYPTION_PASSWORD_PLAINTEXT is not None:
                try:
                    ENCRYPTION_PASSWORD = get_aes_key(ENCRYPTION_PASSWORD_PLAINTEXT)
                except InvalidPasswordError:
                    logging.error("read password from ecs but the password is wrong")
                    ENCRYPTION_PASSWORD_PLAINTEXT = None

    ENCRYPTION_INITIALIZED = True

    GUI_WHITELIST_EXCLUDED_OBSERVABLE_TYPES = [_.strip() for _ in 
                                               CONFIG['gui']['whitelist_excluded_observable_types'].split(',')]

    for o_type in GUI_WHITELIST_EXCLUDED_OBSERVABLE_TYPES:
        if o_type not in VALID_OBSERVABLE_TYPES:
            logging.error(f"invalid observable type {o_type} specified in [gui] whitelist_excluded_observable_types")

    # make this a faster lookup
    GUI_WHITELIST_EXCLUDED_OBSERVABLE_TYPES = set(GUI_WHITELIST_EXCLUDED_OBSERVABLE_TYPES)

    # load global SLA settings
    GLOBAL_SLA_SETTINGS = SLA(None, 
                              CONFIG['SLA'].getboolean('enabled'),
                              CONFIG['SLA'].getint('time_to_dispo'),
                              CONFIG['SLA'].getint('approaching_warn'),
                              None, None)

    EXCLUDED_SLA_ALERT_TYPES = [x.strip() for x in CONFIG['SLA']['excluded_alert_types'].split(',')]

    # load all the other SLA settings
    for section in [s for s in CONFIG.keys() if s.startswith('SLA_')]:
        logging.debug("loading {}".format(section))
        OTHER_SLA_SETTINGS.append(SLA(section[len('SLA_'):],
                                      CONFIG[section].getboolean('enabled'),
                                      CONFIG[section].getint('time_to_dispo'),
                                      CONFIG[section].getint('approaching_warn'),
                                      CONFIG[section]['property'],
                                      CONFIG[section]['value']))

    # what node is this?
    try:
        SAQ_NODE = CONFIG['global']['node']
        if SAQ_NODE == 'AUTO':
            SAQ_NODE = socket.getfqdn()
    except Exception as e:
        sys.stderr.write("unable to get hostname: {}\n".format(e))
        sys.exit(1)

    # what prefix do other systems use to communicate to the API server for this node?
    try:
        API_PREFIX = CONFIG['api']['prefix']
        if API_PREFIX == 'AUTO':
            API_PREFIX = socket.getfqdn()
        logging.debug("node {} has api prefix {}".format(SAQ_NODE, API_PREFIX))
    except Exception as e:
        sys.stderr.write("unable to get hostname: {}\n".format(e))
        sys.exit(1)

    # what type of instance is this?
    if 'instance_type' in CONFIG['global']:
        INSTANCE_TYPE = CONFIG['global']['instance_type']
        if INSTANCE_TYPE not in [ INSTANCE_TYPE_PRODUCTION, INSTANCE_TYPE_QA, INSTANCE_TYPE_DEV ]:
            logging.warning("invalid instance type {}: defaulting to {}".format(INSTANCE_TYPE, INSTANCE_TYPE_PRODUCTION))
            INSTANCE_TYPE = INSTANCE_TYPE_PRODUCTION
    else:
        logging.warning("missing configuration instance_type in global section (defaulting to instance type {})".format(INSTANCE_TYPE_PRODUCTION))

    if FORCED_ALERTS: # lol
        logging.warning(" ****************************************************************** ")
        logging.warning(" ****************************************************************** ")
        logging.warning(" **** WARNING **** ALL ANALYSIS RESULTS IN ALERTS **** WARNING **** ")
        logging.warning(" ****************************************************************** ")
        logging.warning(" ****************************************************************** ")

    # warn if timezone is not UTC
    #if time.strftime("%z") != "+0000":
        #logging.warning("Timezone is not UTC. All ACE systems in a cluster should be in UTC.")

    # we can globally disable semaphores with this flag
    SEMAPHORES_ENABLED = CONFIG.getboolean('global', 'enable_semaphores')

    # some settings can be set to PROMPT
    for section in CONFIG.sections():
        for (name, value) in CONFIG.items(section):
            if value == 'PROMPT':
                CONFIG.set(section, name, getpass("Enter the value for {0}:{1}: ".format(section, name)))

    # make sure we've got the ca chain for SSL certs
    CA_CHAIN_PATH = os.path.join(SAQ_HOME, CONFIG['SSL']['ca_chain_path'])
    ace_api.set_default_ssl_ca_path(CA_CHAIN_PATH)

    # initialize the database connection
    initialize_database()

    # initialize fallback semaphores
    initialize_fallback_semaphores()

    # XXX get rid of this
    try:
        maliciousdir = CONFIG.get("global", "malicious")
    except:
        maliciousdir = "malicious"

    STATS_DIR = os.path.join(DATA_DIR, 'stats')
    MODULE_STATS_DIR = os.path.join(STATS_DIR, 'modules')

    # make sure some key directories exists
    for dir_path in [ 
        os.path.join(DATA_DIR, CONFIG['global']['node']),
        os.path.join(DATA_DIR, 'review', 'rfc822'),
        os.path.join(DATA_DIR, 'review', 'misc'),
        os.path.join(DATA_DIR, CONFIG['global']['error_reporting_dir']),
        STATS_DIR,
        MODULE_STATS_DIR,
        os.path.join(STATS_DIR, 'brocess'), # get rid of this
        os.path.join(STATS_DIR, 'metrics'),
        os.path.join(DATA_DIR, CONFIG['splunk_logging']['splunk_log_dir']),
        os.path.join(DATA_DIR, CONFIG['elk_logging']['elk_log_dir']),
        os.path.join(TEMP_DIR),
        SERVICES_DIR,
        DAEMON_DIR, ]: 
        #os.path.join(SAQ_HOME, maliciousdir) ]: # XXX remove
        try:
            create_directory(dir_path)
        except Exception as e:
            logging.error("unable to create required directory {}: {}".format(dir_path, str(e)))
            sys.exit(1)

    # clear out any proxy environment variables if they exist
    for proxy_key in [ 'http_proxy', 'https_proxy', 'ftp_proxy' ]:
        if proxy_key in os.environ:
            logging.debug("removing proxy environment variable for {}".format(proxy_key))
            del os.environ[proxy_key]

    # set up the PROXY global dict (to be used with the requests library)
    for proxy_key in [ 'http', 'https' ]:
        if CONFIG['proxy']['host'] and CONFIG['proxy']['port'] and CONFIG['proxy']['transport']:
            if CONFIG['proxy']['user'] and CONFIG['proxy']['password']:
                PROXIES[proxy_key] = '{}://{}:{}@{}:{}'.format(
                CONFIG['proxy']['transport'], 
                urllib.parse.quote_plus(CONFIG['proxy']['user']), 
                urllib.parse.quote_plus(CONFIG['proxy']['password']), 
                CONFIG['proxy']['host'], 
                CONFIG['proxy']['port'])
            else:
                PROXIES[proxy_key] = '{}://{}:{}'.format(CONFIG['proxy']['transport'], CONFIG['proxy']['host'], CONFIG['proxy']['port'])

            logging.debug("proxy for {} set to {}".format(proxy_key, PROXIES[proxy_key]))

    # load any additional proxies specified in the config sections proxy_*
    for section in CONFIG.keys():
        if section.startswith('proxy_'):
            proxy_name = section[len('proxy_'):]
            OTHER_PROXIES[proxy_name] = {}
            for proxy_key in [ 'http', 'https' ]:
                if CONFIG[section]['host'] and CONFIG[section]['port'] and CONFIG[section]['transport']:
                    if 'user' in CONFIG[section] and 'password' in CONFIG[section] \
                    and CONFIG[section]['user'] and CONFIG[section]['password']:
                        OTHER_PROXIES[proxy_name][proxy_key] = '{}://{}:{}@{}:{}'.format(
                        CONFIG[section]['transport'], 
                        urllib.parse.quote_plus(CONFIG[section]['user']), 
                        urllib.parse.quote_plus(CONFIG[section]['password']), 
                        CONFIG[section]['host'], 
                        CONFIG[section]['port'])
                    else:
                        OTHER_PROXIES[proxy_name][proxy_key] = '{}://{}:{}'.format(
                        CONFIG[section]['transport'], CONFIG[section]['host'], CONFIG[section]['port'])

    # load global constants
    import iptools
    
    MANAGED_NETWORKS = []
    for cidr in CONFIG['network_configuration']['managed_networks'].split(','):
        try:
            if cidr:
                MANAGED_NETWORKS.append(iptools.IpRange(cidr.strip()))
        except Exception as e:
            logging.error("invalid managed network {}: {}".format(cidr, str(e)))

    # are we running as a daemon?
    if args:
        DAEMON_MODE = args.daemon

    # make sure we've got the automation user set up
    initialize_automation_user()

    # initialize other systems
    initialize_message_system()

    logging.debug("SAQ initialized")