Пример #1
0
def apimethod_run_nmap_scan(sensor_id, target, idm, scan_type, rdns, scan_timing, autodetect, scan_ports,
                            output_file_prefix="", save_to_file=False, job_id=""):
    """Launches an MAP scan
    Args:
        sensor_id: The system IP where you want to get the [sensor]/interfaces from ossim_setup.conf
        target: IP address of the component where the NMAP will be executed
        idm: Convert results into idm events
        scan_type: Sets the NMAP scan type
        rdns: Tells Nmap to do reverse DNS resolution on the active IP addresses it finds
        scan_timing: Set the timing template
        autodetect: Aggressive scan options (enable OS detection)
        scan_ports: Only scan specified ports
        output_file_prefix: Prefix string to be added to the output filename
        save_to_file: Indicates whether you want to save the NMAP report to a file or not.
        job_id: Celery job ID.

    Returns:
        nmap_report: The NMAP report or the filename where the report has been saved.

    Raises:
        APINMAPScanCannotRun
        APICannotResolveSensorID
        APINMAPScanCannotRetrieveBaseFolder
        APINMAPScanCannotCreateLocalFolder
    """
    (result, sensor_ip) = get_sensor_ip_from_sensor_id(sensor_id, local_loopback=False)
    if result is False:
        api_log.error(
            "[apimethod_run_nmap_scan] Cannot retrieve the sensor ip from the given sensor id <%s>" % sensor_id)
        raise APICannotResolveSensorID(sensor_id)
    success, nmap_report = ansible_run_nmap_scan(sensor_ip=sensor_ip, target=target, scan_type=scan_type, rdns=rdns,
                                                 scan_timing=scan_timing, autodetect=autodetect, scan_ports=scan_ports,
                                                 job_id=job_id)
    if not success:
        api_log.error('Failed to launch NMAP scan: %s' % nmap_report)
        raise APINMAPScanCannotRun(nmap_report)

    filename = None
    if save_to_file:
        base_path = get_nmap_directory(sensor_id)
        filename = "%s/nmap_report_%s.json" % (base_path, output_file_prefix)
        with open(filename, "w") as f:
            f.write(json.dumps(nmap_report))

    if idm:
        conn = IDMConnection(sensor_id=sensor_id)
        if conn.connect():
            conn.send_events_from_hosts(nmap_report)
            try:
                if filename is not None:
                    os.remove(filename)
            except Exception:
                pass
        else:
            api_log.error("[apimethod_run_nmap_scan] Cannot connect with the IDM Service")
    try:
        apimethods_nmap_purge_scan_files(job_id)
    except Exception as exp:
        api_log.warning("[apimethod_run_nmap_scan] Cannot purge the scan files %s" % str(exp))
    return nmap_report
Пример #2
0
    def get_pulse_updates(self):
        """Update the redis with the pulses that must been re-added and deleted.
        Args:
            None
        Returns:
            tuple: Number of pulses updated and deleted.
        """
        total_add = 0
        total_del = 0
        subscribed_timestamp = self.get_latest_request('subscribed')
        events_timestamp = self.get_latest_request('events')

        #If it is the first time we download the pulses we don't execute this call.
        if subscribed_timestamp is not None:
            #Getting event time or subscribed time in case event time is null by any reason.
            events_timestamp = subscribed_timestamp if events_timestamp is None else events_timestamp
            next_request = "%s/pulses/events?limit=20&since=%s" % (
                self.url_base, events_timestamp)
        else:
            return total_add, total_del

        event = {}
        while next_request:
            try:
                json_data = self.make_request(next_request)
                #We need to apply the action in each iteration to keep the order of each modification.
                for event in json_data.get('results'):
                    e_type = event.get('object_type')
                    e_action = event.get('action')
                    e_id = event.get('object_id')
                    #Authors to delete
                    if e_type == 'user' and e_action in [
                            'unsubscribe', 'delete'
                    ]:
                        total_del += self.remove_pulses_from_authors([e_id])
                    #Authors to subscribe
                    elif e_type == 'user' and e_action == 'subscribe':
                        total_add += self.add_pulses_from_authors([e_id])
                    #Pulses to delete
                    elif e_type == 'pulse' and e_action in [
                            'unsubscribe', 'delete'
                    ]:
                        total_del += self.remove_pulses([e_id])
                    #Pulses to add
                    elif e_type == 'pulse' and e_action == 'subscribe':
                        total_add += self.add_pulses_from_list([e_id])

                # Get next request
                next_request = json_data.get('next')

            except Exception as error:
                api_log.warning("Cannot download pulse updates: %s" %
                                str(error))
                raise

        update_timestamp = event.get('created', None)
        if update_timestamp is not None:
            self.update_latest_request('events', update_timestamp)

        return total_add, total_del
Пример #3
0
def is_task_in_rabbit(task_id):
    """Checks if the task is in rabbit or not. If the task is found returns the json data, otherwise returns none
    {u'retries': 0, u'task': u'celerymethods.tasks.monitor_tasks.monitor_retrieves_remote_info', u'eta': None, u'args': [], u'expires': None, u'callbacks': None, u'errbacks': None, u'kwargs': {}, u'id': u'de1cd3b1-d001-4bea-a050-8ffe610bee21', u'utc': False}
    """
    try:
        conn = amqp.Connection(host="localhost:5672 ", userid="guest", password="******", virtual_host="/", insist=False)
        chan = conn.channel()

        # Inspect all available queues
        for celery_queue in CELERY_QUEUES:
            while True:
                msg = chan.basic_get(queue=celery_queue.name)
                if msg is not None:
                    try:
                        task_json = json.loads(msg.body)
                        if task_json['id'] == task_id:
                            api_log.warning("Task found in rabbit... that means celery is busy..")
                            return task_json
                    except Exception as cannot_parse:
                        api_log.warning("Cannot parse rabbit message: %s" % str(cannot_parse))
                else:
                    break
    except Exception as e:
        api_log.error("Cannot inspect rabbitmq: %s" % str(e))
    return None
Пример #4
0
def get_sensor_plugins_enabled_by_asset(sensor_id, asset_id=None, no_cache=False):
    """ Get the list of plugins enabled in a sensor by asset
    Params:
        sensor_id (UUID): sensor id
        asset_id (UUID): filter for a specific asset
    Return:
        dictionary with the plugins enabled by asset in the sensor
        filtered by asset_id if provided
    Raises:
        APICannotResolveSensorID
        APICannotGetSensorPlugins
    """
    asset_plugins = {}
    sensor_data = get_sensor_plugins(sensor_id=sensor_id,
                                     no_cache=no_cache)
    if 'enabled' in sensor_data:
        asset_plugins = sensor_data['enabled'].get('devices', {})
    if asset_id is not None:
        asset_plugins = dict((key, value) for key, value in asset_plugins.iteritems() if key == asset_id)

    # Fill the plugin info
    plugins = {}
    for (asset, plugin_list) in asset_plugins.iteritems():
        for plugin in plugin_list:
            if plugin in sensor_data['plugins']:
                if asset not in plugins:
                    plugins[asset] = {}
                plugins[asset][plugin] = sensor_data['plugins'][plugin]
            else:
                api_log.warning("[get_sensor_plugins_enabled_by_asset] "
                                "plugin '{0}' enabled in asset '{1}' in sensor '{2}' Not found".format(
                                    plugin, asset, sensor_id))
    return plugins
Пример #5
0
    def add_pulses_from_authors(self, authors):
        """Add the pulses from some given authors.
        Args:
            authors(list): List of authors we want their pulses to be added
        Returns:
            integer: Number of new pulses added.
        """
        pulse_downloaded = 0
        for author in authors:
            next_request = "{}/pulses/subscribed?limit=20&author_name={}".format(
                self.url_base, author)
            while next_request:
                try:
                    json_data = self.make_request(next_request)
                    # Save pulse data on redis
                    pulse_downloaded += self.save_pulses(
                        json_data.get('results'))
                    # Get next request
                    next_request = json_data.get('next')
                except Exception as err:
                    api_log.warning(
                        "Cannot download pulses from author {}: {}".format(
                            author, err))
                    continue

        return pulse_downloaded
Пример #6
0
def get_sensor_plugins_enabled_by_asset(sensor_id, asset_id=None, no_cache=False):
    """ Get the list of plugins enabled in a sensor by asset
    Params:
        sensor_id (UUID): sensor id
        asset_id (UUID): filter for a specific asset
    Return:
        dictionary with the plugins enabled by asset in the sensor
        filtered by asset_id if provided
    Raises:
        APICannotResolveSensorID
        APICannotGetSensorPlugins
    """
    asset_plugins = {}
    sensor_data = get_sensor_plugins(sensor_id=sensor_id,
                                     no_cache=no_cache)
    if 'enabled' in sensor_data:
        asset_plugins = sensor_data['enabled'].get('devices', {})
    if asset_id is not None:
        asset_plugins = dict((key, value) for key, value in asset_plugins.iteritems() if key == asset_id)

    # Fill the plugin info
    plugins = {}
    for (asset, plugin_list) in asset_plugins.iteritems():
        for plugin in plugin_list:
            if plugin in sensor_data['plugins']:
                if asset not in plugins:
                    plugins[asset] = {}
                plugins[asset][plugin] = sensor_data['plugins'][plugin]
            else:
                api_log.warning("[get_sensor_plugins_enabled_by_asset] "
                                "plugin '{0}' enabled in asset '{1}' in sensor '{2}' Not found".format(
                                    plugin, asset, sensor_id))
    return plugins
Пример #7
0
def apimethod_delete_nmap_scan(sensor_id, task_id):
    """
    Args:
        sensor_id
        task_id
    Returns:

    Raises:
        APINMAPScanCannotRetrieveBaseFolder
        APINMAPScanReportNotFound
        APINMAPScanCannotCreateLocalFolder
        APINMAPScanReportCannotBeDeleted
    """
    try:
        # When the NMAP scan has been stopped by the user, it could leave some files in tmp folder.
        apimethods_nmap_purge_scan_files(task_id)
    except Exception as exp:
        api_log.warning("[apimethod_delete_nmap_scan] Cannot purge the scan files %s" % str(exp))
    apimethod_nmapdb_delete_task(task_id)
    directory = get_nmap_directory(sensor_id)
    nmap_report_path = "{0}/nmap_report_{1}.json".format(directory, task_id)

    # if not os.path.isfile(nmap_report_path):
    #     raise APINMAPScanReportNotFound(nmap_report_path)

    try:
        if os.path.isfile(nmap_report_path):
            os.remove(nmap_report_path)
    except Exception as e:
        api_log.error("[apimethod_delete_nmap_scan] {0}".format(str(e)))
        raise APINMAPScanReportCannotBeDeleted()
    def __get_software_version(self, sensor_id_bin):
        try:
            version = db.session.query(Sensor_Properties.version).filter(
                Sensor_Properties.sensor_id == sensor_id_bin
            ).one()[0]
        except Exception as exc:
            api_log.warning('Did not manage to get sensor software version: {}'.format(format_exc(exc)))
            return None

        return version
Пример #9
0
def is_task_in_celery(task_id):
    """Look whether a task is scheduled, reserved or active
    Args:
        The task id
    Returns the task dictionary or None
    """
    try:
        # When celery is down, inspect will be None, in this case we will wait for a while.
        i = None
        tries = 0
        while tries < 3:
            try:
                i = inspect(timeout=10)
                if inspect is not None:
                    break
            except Exception as exp:
                api_log.warning("Cannot inspect the celery queue.. let's wait for while... %s" % str(exp))
            finally:
                tries = tries + 1
                time.sleep(5)
        if inspect is None:
            return None
        active = i.active()
        scheduled = i.scheduled()
        reserved = i.reserved()
        if active is not None:
            for node, tasks in active.iteritems():
                for task in tasks:
                    if str(task['id']) == task_id:
                        del i
                        return task.copy()
        if reserved is not None:
            for node, tasks in reserved.iteritems():
                for task in tasks:
                    if str(task['id']) == task_id:
                        del i
                        return task.copy()

        if scheduled is not None:
            for node, tasks in scheduled.iteritems():
                for task in tasks:
                    if str(task['id']) == task_id:
                        del i
                        return task.copy()
        del i
        # Wow, we have reached this point...
        # Maybe celery is to busy to get tasks from the queue, let's see whether the task is in rabbit.
        task_in_rabbit = is_task_in_rabbit(task_id)
        if task_in_rabbit is not None:
            return task_in_rabbit

    except Exception as exp:
        api_log.error("[is_task_in_celery] An error occurred while reading the task list %s" % str(exp))

    return None
Пример #10
0
def is_task_in_celery(task_id):
    """Look whether a task is scheduled, reserved or active
    Args:
        The task id
    Returns the task dictionary or None
    """
    try:
        # When celery is down, inspect will be None, in this case we will wait for a while.
        i = None
        tries = 0
        while tries < 3:
            try:
                i = inspect(timeout=10)
                if inspect is not None:
                    break
            except Exception as exp:
                api_log.warning("Cannot inspect the celery queue.. let's wait for while... %s" % str(exp))
            finally:
                tries = tries + 1
                time.sleep(5)
        if inspect is None:
            return None
        active = i.active()
        scheduled = i.scheduled()
        reserved = i.reserved()
        if active is not None:
            for node, tasks in active.iteritems():
                for task in tasks:
                    if str(task['id']) == task_id:
                        del i
                        return task.copy()
        if reserved is not None:
            for node, tasks in reserved.iteritems():
                for task in tasks:
                    if str(task['id']) == task_id:
                        del i
                        return task.copy()

        if scheduled is not None:
            for node, tasks in scheduled.iteritems():
                for task in tasks:
                    if str(task['id']) == task_id:
                        del i
                        return task.copy()
        del i
        # Wow, we have reached this point...
        # Maybe celery is to busy to get tasks from the queue, let's see whether the task is in rabbit.
        task_in_rabbit = is_task_in_rabbit(task_id)
        if task_in_rabbit is not None:
            return task_in_rabbit

    except Exception as exp:
        api_log.error("[is_task_in_celery] An error occurred while reading the task list %s" % str(exp))

    return None
Пример #11
0
    def get_pulse_updates(self):
        """Update the redis with the pulses that must been re-added and deleted.
        Args:
            None
        Returns:
            tuple: Number of pulses updated and deleted.
        """
        total_add = 0
        total_del = 0
        subscribed_timestamp = self.get_latest_request('subscribed')
        events_timestamp = self.get_latest_request('events')

        #If it is the first time we download the pulses we don't execute this call.
        if subscribed_timestamp is not None:
            #Getting event time or subscribed time in case event time is null by any reason.
            events_timestamp = subscribed_timestamp if events_timestamp is None else events_timestamp
            next_request = "%s/pulses/events?limit=20&since=%s" % (self.url_base, events_timestamp)
        else:
            return total_add, total_del

        event = {}
        while next_request:
            try:
                json_data = self.make_request(next_request)
                #We need to apply the action in each iteration to keep the order of each modification.
                for event in json_data.get('results'):
                    e_type = event.get('object_type')
                    e_action = event.get('action')
                    e_id = event.get('object_id')
                    #Authors to delete
                    if e_type == 'user' and e_action in ['unsubscribe', 'delete']:
                        total_del += self.remove_pulses_from_authors([e_id])
                    #Authors to subscribe
                    elif e_type == 'user' and e_action == 'subscribe':
                        total_add += self.add_pulses_from_authors([e_id])
                    #Pulses to delete
                    elif e_type == 'pulse' and e_action in ['unsubscribe', 'delete']:
                        total_del += self.remove_pulses([e_id])
                    #Pulses to add
                    elif e_type == 'pulse' and e_action == 'subscribe':
                        total_add += self.add_pulses_from_list([e_id])

                # Get next request
                next_request = json_data.get('next')

            except Exception as error:
                api_log.warning("Cannot download pulse updates: %s" % str(error))
                raise

        update_timestamp = event.get('created', None)
        if update_timestamp is not None:
            self.update_latest_request('events', update_timestamp)

        return total_add, total_del
Пример #12
0
    def __get_software_version(self, sensor_id_bin):
        try:
            version = db.session.query(Sensor_Properties.version).filter(
                Sensor_Properties.sensor_id == sensor_id_bin).one()[0]
        except Exception as exc:
            api_log.warning(
                'Did not manage to get sensor software version: {}'.format(
                    format_exc(exc)))
            return None

        return version
Пример #13
0
 def _sanitize_tasks(self):
     # Special case for telemetry
     if Task.TELEMERY not in self._custom_tasks:
         try:
             success, value = db_get_config('track_usage_information')
             if success and value != '':
                 telemetry_task = self.get_task(Task.TELEMERY)
                 telemetry_task.enabled = bool(int(value))
                 self.update_task(telemetry_task)
         except Exception as e:
             api_log.warning("[Scheduler._sanitize_tasks] {0}".format(str(e)))
Пример #14
0
    def process(self):
        """Processes the plugin checks"""

        if not self.__plugin_loaded:
            self.__load_plugin()
        try:
            data = self.__plugin_file.check()
        except Exception as e:
            api_log.warning("[PluginTester] Cannot check the plugin %s" % str(e))
            raise APICannotCheckPlugin(self.__plugin_file_name)
        return data
Пример #15
0
 def _sanitize_tasks(self):
     # Special case for telemetry
     if Task.TELEMERY not in self._custom_tasks:
         try:
             success, value = db_get_config('track_usage_information')
             if success and value != '':
                 telemetry_task = self.get_task(Task.TELEMERY)
                 telemetry_task.enabled = bool(int(value))
                 self.update_task(telemetry_task)
         except Exception as e:
             api_log.warning("[Scheduler._sanitize_tasks] {0}".format(
                 str(e)))
Пример #16
0
    def process(self):
        """Processes the plugin checks"""

        if not self.__plugin_loaded:
            self.__load_plugin()
        try:
            data = self.__plugin_file.check()
        except Exception as e:
            api_log.warning("[PluginTester] Cannot check the plugin %s" %
                            str(e))
            raise APICannotCheckPlugin(self.__plugin_file_name)
        return data
Пример #17
0
    def unregister_console(self):
        token = self.__token_repository.get_token()
        if token is None:
            raise Exception('unregister_console failed: token was not found')

        try:
            self.__console_proxy.send_disconnect_notification(token)
        except Exception as exc:
            # Disconnect notification is not obligatory, so it should not block the unregistration.
            # This allows to unregister from the console which is not accessible without extending
            # business logic with something like 'force unregister'.
            api_log.warning('Disconnect notification failed, ignoring. Error details: {}'.format(format_exc(exc)))
        finally:
            self.__token_repository.delete_token()
            return CentralConsoleStatus(CONSOLE_CONNECTION_NOT_CONFIGURED, None)
Пример #18
0
    def get_new_pulses(self):
        """Update the redis with the pulses that must been added.
        Args:
            None
        Returns:
            integer: Number of new pulses downloaded.
        """
        pulse_downloaded = 0
        subscribed_timestamp = self.get_latest_request('subscribed')

        if subscribed_timestamp is not None:
            next_request = "{}/pulses/subscribed?limit=20&modified_since={}".format(
                self.url_base, subscribed_timestamp)
        else:
            next_request = "{}/pulses/subscribed?limit=20".format(
                self.url_base)

        # This var will store the date of the newest pulse that will be used to query the next time.
        update_timestamp = None
        while next_request:
            try:
                json_data = self.make_request(next_request)
                p_data = json_data.get('results', [])
                # First we remove the pulse to avoid IOC inconsistency problems.
                self.remove_pulses([p.get('id', '') for p in p_data])
                # Save pulse data on redis
                pulse_downloaded += self.save_pulses(p_data)
                # Save the newest pulse date
                if update_timestamp is None:
                    try:
                        # We save the first pulse modified date.
                        update_timestamp = p_data[0]['modified']
                    except:
                        pass
                # Get next request
                next_request = json_data.get('next')
            except Exception as err:
                api_log.warning("Cannot download new pulses: {}".format(err))
                raise

        # Saving the request date
        if update_timestamp is not None:
            self.update_latest_request('subscribed', update_timestamp)
        # If it is the first time we download the pulses, we update the event request time to the current UTC timestamp.
        if subscribed_timestamp is None:
            self.update_latest_request('events')

        return pulse_downloaded
Пример #19
0
    def get_otx_user_version():
        """ Returns string like 'OTX USM/5.2.4' if able to get system version or '' otherwise
        """
        otx_user_version = ''
        try:
            data_retrieved, version_data = get_alienvault_version()
            if data_retrieved:
                # we need only product name and version number, so get first 2
                version_data = version_data.replace('ALIENVAULT', 'USM').split()[:2]
                otx_user_version = 'OTX {}'.format('/'.join(version_data))
            else:
                api_log.warning('Bad result returned for alienvault version: {}'.format(version_data))
        except Exception as err:
            api_log.warning('Failed to get alienvault version. Reason: {}'.format(err))

        return otx_user_version
Пример #20
0
    def check_token(self):
        """Checks if a OTX token is valid and return user info if so.
        Args:
            None
        Returns:
            user_data(dict): A dict with the user info.
        """
        url = "{}/user/".format(self.url_base)

        try:
            user_data = self.make_request(url)
        except Exception as err:
            api_log.warning("OTX key activation error: {}".format(err))
            raise

        return user_data
Пример #21
0
    def check_token(self):
        """Checks if a OTX token is valid and return user info if so.
        Args:
            None
        Returns:
            user_data(dict): A dict with the user info.
        """
        url = "%s/user/" % self.url_base

        try:
            user_data = self.make_request(url)
        except Exception as error:
            api_log.warning("OTX key activation error: %s" % str(error))
            raise

        return user_data
Пример #22
0
def has_admin_users():
    """
    Tells if there are any admin users in the database or not.
    Returns:
        result(bool):True if success, False otherwise
    """
    result = False
    try:
        result = db.session.query(Config).filter(Config.conf == 'first_login').one().value.lower() == 'no'
    except NoResultFound:
        result = False
        api_log.warning("[has_admin_users] No first_login row found.")
    except Exception as error:
        api_log.error("[has_admin_users] %s" % str(error))
        result = False
    return result
Пример #23
0
    def get_new_pulses(self):
        """Update the redis with the pulses that must been added.
        Args:
            None
        Returns:
            integer: Number of new pulses downloaded.
        """
        pulse_downloaded = 0
        subscribed_timestamp = self.get_latest_request('subscribed')

        if subscribed_timestamp is not None:
            next_request = "%s/pulses/subscribed?limit=20&modified_since=%s" % (self.url_base, subscribed_timestamp)
        else:
            next_request = "%s/pulses/subscribed?limit=20" % self.url_base

        #This var will store the date of the newest pulse that will be used to query the next time.
        update_timestamp = None
        while next_request:
            try:
                json_data = self.make_request(next_request)
                p_data = json_data.get('results', [])
                # First we remove the pulse to avoid IOC inconsistency problems.
                self.remove_pulses([p.get('id', '') for p in p_data])
                # Save pulse data on redis
                pulse_downloaded += self.save_pulses(p_data)
                #Save the newest pulse date
                if update_timestamp is None:
                    try:
                        #We save the first pulse modified date.
                        update_timestamp = p_data[0]['modified']
                    except:
                        pass
                # Get next request
                next_request = json_data.get('next')
            except Exception as error:
                api_log.warning("Cannot download new pulses: %s" % str(error))
                raise

        #Saving the request date
        if update_timestamp is not None:
            self.update_latest_request('subscribed', update_timestamp)
        #If it is the first time we download the pulses, we update the event request time to the current UTC timestamp.
        if subscribed_timestamp is None:
            self.update_latest_request('events')

        return pulse_downloaded
Пример #24
0
def has_admin_users():
    """
    Tells if there are any admin users in the database or not.
    Returns:
        result(bool):True if success, False otherwise
    """
    result = False
    try:
        result = db.session.query(Config).filter(
            Config.conf == 'first_login').one().value.lower() == 'no'
    except NoResultFound:
        result = False
        api_log.warning("[has_admin_users] No first_login row found.")
    except Exception as error:
        api_log.error("[has_admin_users] %s" % str(error))
        result = False
    return result
Пример #25
0
    def get_latest_request(self, d_type):
        """Loads the latest request timestamp
        Args:
            type (str): The type of date to get.
        Returns:
            None or the date in string format
        """
        date_type = self.date_types.get(d_type, None)
        if date_type is None:
            return date_type

        try:
            latest_timestamp = self.pulse_db.get(date_type)
        except Exception as err:
            api_log.warning("Cannot get messages revision: {}".format(err))
            return None

        return None if latest_timestamp == "" else latest_timestamp
Пример #26
0
    def get_latest_request(self, d_type):
        """Loads the latest request timestamp
        Args:
            type (str): The type of date to get.
        Returns:
            None or the date in string format
        """
        date_type = self.date_types.get(d_type, None)
        if date_type is None:
            return date_type

        try:
            latest_timestamp = self.pulse_db.get(date_type)
        except Exception as err:
            api_log.warning("Cannot get messages revision: %s" % str(err))
            return None

        return None if latest_timestamp == "" else latest_timestamp
Пример #27
0
    def add_pulses_from_list(self, pulses):
        """Add the pulses given.
        Args:
            pulses(list): List of pulses we want to add
        Returns:
            integer: Number of new pulses downloaded.
        """
        p_download = []
        for p_id in pulses:
            request = "%s/pulses/%s/" % (self.url_base, p_id)
            try:
                json_data = self.make_request(request)
                # Save pulse data on redis
                p_download.append(json_data)
            except Exception as error:
                api_log.warning("Cannot download pulse %s: %s" % (str(p_id), str(error)))
                continue

        return self.save_pulses(p_download)
Пример #28
0
    def cache_func(*args, **kwargs):
        cache = _get_cache(namespace=namespace, expire=expire)

        # Handle cases where caching is down or otherwise not available.
        if cache.connection is None:
            return function(*args, **kwargs)

        # Clean kwargs. Remove no_cache and convert str to unicode
        key_kwargs = kwargs.copy()
        if 'no_cache' in kwargs:
            key_kwargs.pop('no_cache')
        for key in key_kwargs:
            if isinstance(key_kwargs[key], str):
                try:
                    key_kwargs[key] = unicode(key_kwargs[key])
                except Exception as e:
                    api_log.warning("use_cache: %s" % str(e))
        # Clean args. convert str to unicode
        key_args = []
        for arg in args:
            if isinstance(arg, str):
                unicode_arg = arg
                try:
                    unicode_arg = unicode(arg)
                except Exception as e:
                    api_log.warning("use_cache: %s" % str(e))
                key_args.append(unicode_arg)
            else:
                key_args.append(arg)

        # Use funtion name, args and kwargs for cache key
        cache_key = hashlib.sha256(
            str(function.__name__) + str(key_args) +
            str(key_kwargs)).hexdigest()
        if 'no_cache' not in kwargs or not kwargs['no_cache']:
            try:
                # Consider using pickle instead of json
                # json can't store tuples
                return cache.get_json(cache_key)
            except (ExpiredKeyException, CacheMissException):
                pass
            except Exception, msg:
                api_log.error(str(msg))
Пример #29
0
    def get_otx_user_version():
        """ Returns string like 'OTX USM/5.2.4' if able to get system version or '' otherwise
        """
        otx_user_version = ''
        try:
            data_retrieved, version_data = get_alienvault_version()
            if data_retrieved:
                # we need only product name and version number, so get first 2
                version_data = version_data.replace('ALIENVAULT',
                                                    'USM').split()[:2]
                otx_user_version = 'OTX {}'.format('/'.join(version_data))
            else:
                api_log.warning(
                    'Bad result returned for alienvault version: {}'.format(
                        version_data))
        except Exception as err:
            api_log.warning(
                'Failed to get alienvault version. Reason: {}'.format(err))

        return otx_user_version
Пример #30
0
    def add_pulses_from_list(self, pulses):
        """Add the pulses given.
        Args:
            pulses(list): List of pulses we want to add
        Returns:
            integer: Number of new pulses downloaded.
        """
        p_download = []
        for p_id in pulses:
            request = "{}/pulses/{}/".format(self.url_base, p_id)
            try:
                json_data = self.make_request(request)
                # Save pulse data on redis
                p_download.append(json_data)
            except Exception as err:
                api_log.warning("Cannot download pulse {}: {}".format(
                    p_id, err))
                continue

        return self.save_pulses(p_download)
Пример #31
0
    def cache_func(*args, **kwargs):
        cache = _get_cache(namespace=namespace, expire=expire)

        # Handle cases where caching is down or otherwise not available.
        if cache.connection is None:
            return function(*args, **kwargs)

        # Clean kwargs. Remove no_cache and convert str to unicode
        key_kwargs = kwargs.copy()
        if 'no_cache' in kwargs:
            key_kwargs.pop('no_cache')
        for key in key_kwargs:
            if isinstance(key_kwargs[key], str):
                try:
                    key_kwargs[key] = unicode(key_kwargs[key])
                except Exception as e:
                    api_log.warning("use_cache: %s" % str(e))
        # Clean args. convert str to unicode
        key_args = []
        for arg in args:
            if isinstance(arg, str):
                unicode_arg = arg
                try:
                    unicode_arg = unicode(arg)
                except Exception as e:
                    api_log.warning("use_cache: %s" % str(e))
                key_args.append(unicode_arg)
            else:
                key_args.append(arg)

        # Use funtion name, args and kwargs for cache key
        cache_key = hashlib.sha256(str(function.__name__) + str(key_args) + str(key_kwargs)).hexdigest()
        if 'no_cache' not in kwargs or not kwargs['no_cache']:
            try:
                # Consider using pickle instead of json
                # json can't store tuples
                return cache.get_json(cache_key)
            except (ExpiredKeyException, CacheMissException):
                pass
            except Exception, msg:
                api_log.error(str(msg))
Пример #32
0
def get_plugin_package_info(system_ip):
    """
        If exists, return the md5sum of INSTALLED VERSION of package alienvault-plugin-sid
        IMPORTANT NOTE: The file must be called alienvault-plugin_<version>_all.deb
    """
    (success, version) = get_plugin_package_version(system_ip)
    if success:
        command = """md5sum /var/cache/apt/archives/alienvault-plugin-sids_%s_all.deb|awk {'print $1'}""" % version
        response = ansible.run_module(host_list=[system_ip], module="shell", args=command)
        if system_ip in response['contacted']:
            if response['contacted'][system_ip]['rc'] == 0:
                md5 = response['contacted'][system_ip]['stdout'].split('\n')[0]  # Only first line
            else:
                api_log.warning("Can't obtanin md5 for alienvault-plugin-sids")
                md5 = ''
            result = (True, {'version': version, 'md5': md5})
        else:
            result = (False, str(response['dark'][system_ip]))
    else:
        result = (False, "Can't obtain package version")
    return result
Пример #33
0
    def add_pulses_from_authors(self, authors):
        """Add the pulses from some given authors.
        Args:
            authors(list): List of authors we want their pulses to be added
        Returns:
            integer: Number of new pulses added.
        """
        pulse_downloaded = 0
        for author in authors:
            next_request = "%s/pulses/subscribed?limit=20&author_name=%s" % (self.url_base, author)
            while next_request:
                try:
                    json_data = self.make_request(next_request)
                    # Save pulse data on redis
                    pulse_downloaded += self.save_pulses(json_data.get('results'))
                    # Get next request
                    next_request = json_data.get('next')
                except Exception as error:
                    api_log.warning("Cannot download pulses from author %s: %s" % (str(author), str(error)))
                    continue

        return pulse_downloaded
Пример #34
0
def is_task_in_rabbit(task_id):
    """Checks if the task is in rabbit or not. If the task is found returns the json data, otherwise returns none
    {u'retries': 0, u'task': u'celerymethods.tasks.monitor_tasks.monitor_retrieves_remote_info', u'eta': None, u'args': [], u'expires': None, u'callbacks': None, u'errbacks': None, u'kwargs': {}, u'id': u'de1cd3b1-d001-4bea-a050-8ffe610bee21', u'utc': False}
    """
    try:
        conn = amqp.Connection(host="localhost:5672 ", userid="guest", password="******", virtual_host="/", insist=False)
        chan = conn.channel()
        while True:
            msg = chan.basic_get(queue="celery")
            if msg is not None:
                try:
                    task_json = json.loads(msg.body)
                    if task_json['id'] == task_id:
                        api_log.warning("Task found in rabbit... that means celery is busy..")
                        return task_json
                except Exception as cannot_parse:
                    api_log.warning("Cannot parse rabbit message: %s" % str(cannot_parse))
            else:
                break
    except Exception as e:
        api_log.error("Cannot inspect rabbitmq: %s" % str(e))
    return None
Пример #35
0
    def start(self):
        """ Starts the monitor activity
        """
        try:
            # Remove the previous monitor data.
            self.remove_monitor_data()
            monitor_data = {}
            success, system_id = get_system_id_from_local()
            if not success:
                return False

            # Now
            now = int(time.time())

            # Firstly, wizard data!
            wizard_dict = {}
            success, start_welcome_wizard, welcome_wizard_date = get_wizard_data(
            )
            if not success:
                api_log.error("There was an error retrieving the wizard data")

            wizard_shown = True
            if start_welcome_wizard == 2:
                # if difference between now and welcome_wizard_date is less
                # than a week, display message
                if (now - welcome_wizard_date) < 420:
                    wizard_shown = False

            wizard_dict['wizard_shown'] = wizard_shown
            monitor_data[
                self.__WEB_MESSAGES['MESSAGE_WIZARD_SHOWN']] = wizard_dict

            # Time to look for orphan sensors
            orphan_sensors_dict = {}
            success, message = check_any_orphan_sensor()
            orphan_sensors = False
            if not success:
                api_log.error(message)
                orphan_sensors = True

            orphan_sensors_dict['orphan_sensors'] = orphan_sensors
            monitor_data[self.__WEB_MESSAGES[
                'MESSAGE_SENSOR_NOT_INSERTED']] = orphan_sensors_dict

            # Has the trial version expired?
            success, expires, message = get_trial_expiration_date()
            trial_expired = False
            trial_expires_7days = False
            trial_expires_2days = False
            if not success:
                rc, pro = system_is_professional()
                if rc:
                    if pro:
                        # OK, we have an error here
                        api_log.error(message)
                    else:
                        pass
            else:
                # expire=9999-12-31
                expiration_date = expires.split('=')[1]
                if expiration_date:
                    mktime_expression = datetime.datetime.strptime(
                        expiration_date, "%Y-%m-%d").timetuple()
                    expires = int(time.mktime(mktime_expression))

                    one_week_left = now - 604800
                    two_days_left = now - 172800

                    if expires < one_week_left:
                        trial_expires_7days = True
                    elif expires < two_days_left:
                        trial_expires_2days = True
                    elif expires < now:
                        trial_expired = True
                    else:
                        pass
                else:
                    if os.path.isfile("/etc/ossim/ossim.lic"):
                        api_log.warning(
                            "Valid license but no web admin user found!")
                    else:
                        api_log.debug(
                            "Expiration date can't be determined: License file not found"
                        )

            monitor_data[self.__WEB_MESSAGES["MESSAGE_TRIAL_EXPIRED"]] = {
                'trial_checked': success,
                'trial_expired': trial_expired
            }
            monitor_data[
                self.__WEB_MESSAGES["MESSAGE_TRIAL_EXPIRES_7DAYS"]] = {
                    'trial_checked': success,
                    'trial_expired': trial_expires_7days
                }
            monitor_data[
                self.__WEB_MESSAGES["MESSAGE_TRIAL_EXPIRES_2DAYS"]] = {
                    'trial_checked': success,
                    'trial_expired': trial_expires_2days
                }

            # Check max number of assets
            assets = len(get_asset_list())
            contracted_devices = get_license_devices()
            over_assets = False
            exceeding_assets = 0
            #if assets > contracted_devices:
            #    exceeding_assets = assets - contracted_devices
            #    over_assets = True
            monitor_data[self.__WEB_MESSAGES["MESSAGE_LICENSE_VIOLATION"]] = {
                'over_assets': over_assets,
                'exceeding_assets': exceeding_assets
            }

            # OTX contribution
            otx_enabled = apimethod_is_otx_enabled()
            monitor_data[self.__WEB_MESSAGES["MESSAGE_OTX_CONNECTION"]] = {
                'otx_enabled': otx_enabled
            }

            # Backup in progress?
            success, running, message = check_backup_process_running()
            if not success:
                api_log.error(message)

            monitor_data[self.__WEB_MESSAGES["MESSAGE_BACKUP_RUNNING"]] = {
                'backup_check': success,
                'backup_running': running
            }

            # Save monitor data
            self.save_data(system_id, ComponentTypes.SYSTEM,
                           self.get_json_message(monitor_data))

        except Exception as err:
            api_log.error(
                "Error processing WebUIData monitor information: %s" %
                str(err))
            return False
        return True
Пример #36
0
    def start(self):
        """ Starts the monitor activity
        """
        try:
            # Remove the previous monitor data.
            self.remove_monitor_data()
            monitor_data = {}
            success, system_id = get_system_id_from_local()
            if not success:
                return False

            # Now
            now = int(time.time())

            # Firstly, wizard data!
            wizard_dict = {}
            success, start_welcome_wizard, welcome_wizard_date = get_wizard_data()
            if not success:
                api_log.error("There was an error retrieving the wizard data")

            wizard_shown = True
            if start_welcome_wizard == 2:
                # if difference between now and welcome_wizard_date is less
                # than a week, display message
                if (now - welcome_wizard_date) < 420:
                    wizard_shown = False

            wizard_dict['wizard_shown'] = wizard_shown
            monitor_data[self.__WEB_MESSAGES['MESSAGE_WIZARD_SHOWN']] = wizard_dict

            # Time to look for orphan sensors
            orphan_sensors_dict = {}
            success, message = check_any_orphan_sensor()
            orphan_sensors = False
            if not success:
                api_log.error(message)
                orphan_sensors = True

            orphan_sensors_dict['orphan_sensors'] = orphan_sensors
            monitor_data[self.__WEB_MESSAGES['MESSAGE_SENSOR_NOT_INSERTED']] = orphan_sensors_dict

            # Has the trial version expired?
            success, expires, message = get_trial_expiration_date()
            trial_expired = False
            trial_expires_7days = False
            trial_expires_2days = False
            if not success:
                rc, pro = system_is_professional()
                if rc:
                    if pro:
                        # OK, we have an error here
                        api_log.error(message)
                    else:
                        pass
            else:
                # expire=9999-12-31
                expiration_date = expires.split('=')[1]
                if expiration_date:
                    mktime_expression = datetime.datetime.strptime(expiration_date,
                                                                   "%Y-%m-%d").timetuple()
                    expires = int(time.mktime(mktime_expression))

                    one_week_left = now - 604800
                    two_days_left = now - 172800

                    if expires < one_week_left:
                        trial_expires_7days = True
                    elif expires < two_days_left:
                        trial_expires_2days = True
                    elif expires < now:
                        trial_expired = True
                    else:
                        pass
                else:
                    if os.path.isfile("/etc/ossim/ossim.lic"):
                        api_log.warning("Valid license but no web admin user found!")
                    else:
                        api_log.debug("Expiration date can't be determined: License file not found")

            monitor_data[self.__WEB_MESSAGES["MESSAGE_TRIAL_EXPIRED"]] = {'trial_checked': success,
                                                                          'trial_expired': trial_expired}
            monitor_data[self.__WEB_MESSAGES["MESSAGE_TRIAL_EXPIRES_7DAYS"]] = {'trial_checked': success,
                                                                                'trial_expired': trial_expires_7days}
            monitor_data[self.__WEB_MESSAGES["MESSAGE_TRIAL_EXPIRES_2DAYS"]] = {'trial_checked': success,
                                                                                'trial_expired': trial_expires_2days}

            # Check max number of assets
            assets = len(get_asset_list())
            contracted_devices = get_license_devices()
            over_assets = False
            exceeding_assets = 0
            #if assets > contracted_devices:
            #    exceeding_assets = assets - contracted_devices
            #    over_assets = True
            monitor_data[self.__WEB_MESSAGES["MESSAGE_LICENSE_VIOLATION"]] = {'over_assets': over_assets,
                                                                              'exceeding_assets': exceeding_assets}

            # OTX contribution
            otx_enabled = apimethod_is_otx_enabled()
            monitor_data[self.__WEB_MESSAGES["MESSAGE_OTX_CONNECTION"]] = {'otx_enabled': otx_enabled}

            # Backup in progress?
            success, running, message = check_backup_process_running()
            if not success:
                api_log.error(message)

            monitor_data[self.__WEB_MESSAGES["MESSAGE_BACKUP_RUNNING"]] = {'backup_check': success,
                                                                           'backup_running': running}

            # Save monitor data
            self.save_data(system_id,
                           ComponentTypes.SYSTEM,
                           self.get_json_message(monitor_data))

        except Exception as err:
            api_log.error("Error processing WebUIData monitor information: %s" % str(err))
            return False
        return True
Пример #37
0
def load_mcserver_messages(message_list):
    """Adds or updates messages coming from the mcserver

    Args:
        message_list[Status_Message]

    Returns:
        success (bool): True if successful, False elsewhere
        result  (str): Error message (if any)
    """
    result = ""
    success = True
    try:
        db.session.begin()
        for msg in message_list:
            msg_id_str = str(msg['msg_id'])
            msg_id_binary = get_bytes_from_uuid(msg_id_str)
            additional_info_json = ""
            if msg['additional_info'] is not None and msg['additional_info'] != "":
                try:
                    additional_info_json = json.dumps(msg['additional_info'])
                except Exception as e:
                    api_log.warning("Message with an invalid additional_info %s -  %s" % (msg_id_str, str(e)))
                    additional_info_json = ""
            success, status_message = get_status_message_from_id(message_id=msg_id_binary, is_admin=True,
                                                                 serialize=False)
            if success:
                #update values:
                status_message.level = Status_Message.get_level_integer_from_string(str(msg['level']))
                status_message.title = msg['title']
                status_message.description = msg['description']
                status_message.type = msg['type']
                success, current_status_message = get_current_status_from_message_id(msg_id_str)
                if not success or len(current_status_message) != 1:
                    api_log.error("Invalid external message %s. Current_Status: %s, tuples(%s)" % (
                        msg_id_str, success, len(current_status_message)))
                    continue
                current_status_message[0].additional_info = additional_info_json
                db.session.merge(current_status_message[0])
                db.session.merge(status_message)
            else:
                new_msg = Status_Message()
                new_msg.id = msg_id_binary
                new_msg.level = Status_Message.get_level_integer_from_string(str(msg['level']))
                new_msg.title = msg['title']
                new_msg.description = msg['description']
                new_msg.type = msg['type']
                new_msg.expire = datetime.strptime(msg['valid_to'], "%Y-%m-%dT%H:%M:%S")
                new_msg.actions = ""
                new_msg.alternative_actions = ""
                new_msg.source = "external"
                current_status_message = Current_Status()
                current_status_message.id = uuid4().bytes
                current_status_message.component_type = 'external'
                current_status_message.creation_time = datetime.strptime(msg['valid_from'], "%Y-%m-%dT%H:%M:%S")
                current_status_message.message_id = new_msg.id
                current_status_message.additional_info = ""
                current_status_message.suppressed = 0
                current_status_message.viewed = 0
                current_status_message.additional_info = additional_info_json
                db.session.add(new_msg)
                db.session.add(current_status_message)
        db.session.commit()
    except Exception, e:
        success = False
        result = "[load_mcserver_messages(] Error: %s" % str(e)
        db.session.rollback()
Пример #38
0
def load_mcserver_messages(message_list):
    """Adds or updates messages coming from the mcserver

    Args:
        message_list[Status_Message]

    Returns:
        success (bool): True if successful, False elsewhere
        result  (str): Error message (if any)
    """
    result = ""
    success = True
    try:
        db.session.begin()
        for msg in message_list:
            msg_id_str = str(msg['msg_id'])
            msg_id_binary = get_bytes_from_uuid(msg_id_str)
            additional_info_json = ""
            if msg['additional_info'] is not None and msg[
                    'additional_info'] != "":
                try:
                    additional_info_json = json.dumps(msg['additional_info'])
                except Exception as e:
                    api_log.warning(
                        "Message with an invalid additional_info %s -  %s" %
                        (msg_id_str, str(e)))
                    additional_info_json = ""
            success, status_message = get_status_message_from_id(
                message_id=msg_id_binary, is_admin=True, serialize=False)
            if success:
                #update values:
                status_message.level = Status_Message.get_level_integer_from_string(
                    str(msg['level']))
                status_message.title = msg['title']
                status_message.description = msg['description']
                status_message.type = msg['type']
                success, current_status_message = get_current_status_from_message_id(
                    msg_id_str)
                if not success or len(current_status_message) != 1:
                    api_log.error(
                        "Invalid external message %s. Current_Status: %s, tuples(%s)"
                        % (msg_id_str, success, len(current_status_message)))
                    continue
                current_status_message[
                    0].additional_info = additional_info_json
                db.session.merge(current_status_message[0])
                db.session.merge(status_message)
            else:
                new_msg = Status_Message()
                new_msg.id = msg_id_binary
                new_msg.level = Status_Message.get_level_integer_from_string(
                    str(msg['level']))
                new_msg.title = msg['title']
                new_msg.description = msg['description']
                new_msg.type = msg['type']
                new_msg.expire = datetime.strptime(msg['valid_to'],
                                                   "%Y-%m-%dT%H:%M:%S")
                new_msg.actions = ""
                new_msg.alternative_actions = ""
                new_msg.source = "external"
                current_status_message = Current_Status()
                current_status_message.id = uuid4().bytes
                current_status_message.component_type = 'external'
                current_status_message.creation_time = datetime.strptime(
                    msg['valid_from'], "%Y-%m-%dT%H:%M:%S")
                current_status_message.message_id = new_msg.id
                current_status_message.additional_info = ""
                current_status_message.suppressed = 0
                current_status_message.viewed = 0
                current_status_message.additional_info = additional_info_json
                db.session.add(new_msg)
                db.session.add(current_status_message)
        db.session.commit()
    except Exception, e:
        success = False
        result = "[load_mcserver_messages(] Error: %s" % str(e)
        db.session.rollback()