def is_task_in_rabbit(task_id): """Checks if the task is in rabbit or not. If the task is found returns the json data, otherwise returns none {u'retries': 0, u'task': u'celerymethods.tasks.monitor_tasks.monitor_retrieves_remote_info', u'eta': None, u'args': [], u'expires': None, u'callbacks': None, u'errbacks': None, u'kwargs': {}, u'id': u'de1cd3b1-d001-4bea-a050-8ffe610bee21', u'utc': False} """ try: conn = amqp.Connection(host="localhost:5672 ", userid="guest", password="******", virtual_host="/", insist=False) chan = conn.channel() # Inspect all available queues for celery_queue in CELERY_QUEUES: while True: msg = chan.basic_get(queue=celery_queue.name) if msg is not None: try: task_json = json.loads(msg.body) if task_json['id'] == task_id: api_log.warning("Task found in rabbit... that means celery is busy..") return task_json except Exception as cannot_parse: api_log.warning("Cannot parse rabbit message: %s" % str(cannot_parse)) else: break except Exception as e: api_log.error("Cannot inspect rabbitmq: %s" % str(e)) return None
def apimethod_remove_plugin(plugin_file): """Removes a custom plugin from the systems""" try: plugin_path = os.path.join(END_FOLDER, plugin_file) if not os.path.isfile(plugin_path): raise APIPluginFileNotFound(plugin_file) plugin = PluginFile() # TODO: make some handy wrapper to combine read and check plugin.read(plugin_path, encoding='latin1') plugin.check() # validate and load all the plugins data plugin_data = get_plugin_data_for_plugin_id(plugin.plugin_id) if plugin_data is not None: if plugin_data.plugin_type == PluginDataType.ALIENVAULT_PLUGIN: raise APICannotBeRemoved("This is an AlienVault Plugin. It cannot be removed") # Remove the sids remove_plugin_data(plugin.plugin_id) remove_plugin_from_sensors(plugin_path) # Remove sql file locally (it's located only on server) os.remove(plugin_path + '.sql') except Exception as e: api_log.error("[apimethod_remove_plugin] {}".format(e)) if not isinstance(e, APIException): raise APICannotBeRemoved("{}".format(e)) else: raise
def delete_orphan_hids_agents(agent_list, sensor_id): """ Delete orphan HIDS agents Args: agent_list(list): List of active HIDS agents sensor_id(str): Sensor ID Raises: APICannotResolveSensorID APICannotDeleteHIDSAgentList """ if sensor_id is None: api_log.error("[delete_orphan_hids_agents]: Sensor ID could not be empty") raise APICannotResolveSensorID(sensor_id) try: if agent_list: q_agent_list = "'" + "','".join(agent_list) + "'" sensor_id_hex = get_hex_string_from_uuid(sensor_id) query = "DELETE FROM hids_agents WHERE sensor_id = UNHEX('{0}') " \ "AND agent_id NOT IN ({1})".format(sensor_id_hex, q_agent_list) db.session.connection(mapper=Hids_Agents).execute(query) except Exception as msg: api_log.error("[delete_orphan_hids_agents]: %s" % str(msg)) raise APICannotDeleteHIDSAgentList(agent_list, sensor_id)
def ansible_get_process_pid(system_ip, ps_filter): """Check whether a process is running or not Args: system_ip(str): The system IP where we would like to run the ps filter ps_filter(str): Filter to grep the ps aux command Returns: (boolean,int): A tuple containing whether the operation was well or not and the PID of the process running that meet the filter (0 = not running) """ try: cmd = ('ps aux | grep \"%s\" | grep -v grep | ' 'grep -v tail | tr -s \" \" | cut -d \" \" -f 2 | ' 'head -n 1' % str(re.escape(ps_filter))) response = ansible.run_module(host_list=[system_ip], module="shell", use_sudo="True", args=cmd) (success, msg) = ansible_is_valid_response(system_ip, response) if not success: api_log.error("[ansible_get_process_pid] Error: %s" % str(msg)) return False, 0 pid = response['contacted'][system_ip]['stdout'] if pid: pid = int(pid) else: pid = 0 except Exception as exc: api_log.error("[ansible_get_process_pid] Error: %s" % str(exc)) return False, 0 return success, pid
def get_sensor_ctx_by_sensor_id(sensor_id, output='str'): """ Returns a sensor CTX given a sensor ID """ sensor_ctx = None try: if sensor_id: sensor_id_bin = get_bytes_from_uuid(sensor_id) query = db.session.query(Acl_Sensors.entity_id).filter(Acl_Sensors.sensor_id == sensor_id_bin) sensor = query.join(Acl_Entities, Acl_Entities.id == Acl_Sensors.entity_id).filter( Acl_Entities.entity_type == 'context').one() sensor_ctx = sensor.entity_id else: return False, "Sensor ID could not be empty" except NoResultFound: return True, sensor_ctx except Exception as msg: msg = str(msg) api_log.error(msg) return False, msg if output == 'str': return True, get_uuid_string_from_bytes(sensor_ctx) else: return True, sensor_ctx
def set_file_permissions(filename, mode): try: os.chmod(filename,mode) except Exception as err: api_log.error("Error setting the permissions to the file %s <%s>" % (filename,str(err))) return False, "Error setting the permissions to the file %s <%s>" % (filename,str(err)) return True, ""
def apimethod_check_task_status(system_id, tasks): """ Check the status of a given list of tasks. IE: alienvault-update, alienvault-reconfig Args: system_id (str) : The system_id where you want to check if it's running tasks (dict) : The list of tasks to test. Returns: success (bool) : True if successful, False elsewhere task_status (dict) : A dictionary containing job_id, job_status for each task """ task_status = {} success, system_ip = get_system_ip_from_system_id(system_id) if not success: error_msg = "[apimethod_check_task_status] " + \ "Unable to get system ip " + \ "for system id %s: %s" % (system_id, system_ip) api_log.error(error_msg) return False, {} success, task_status = get_task_status(system_id, system_ip, tasks) if not success: error_msg = "[apimethod_check_task_status] " + \ "Unable to get the task status " + \ "for system %s: %s" % (system_id, str(task_status)) api_log.error(error_msg) return False, {} return success, task_status
def apimethod_get_otx_pulse_stats_event_trend(user_dic, pulse_id='', day_range=0): """Get the trend of events with pulses: Args: user_dic(dic) : User Filter (Login and TZ) pulse_id(str) : ID of the pulse we want to get the trend of events or empty for all pulses day_range(int): Number of days to count - empty means everything Returns: success (bool) : True if successful, False elsewhere trend_list (dict): Error message if there was an error or dic with the trend pulses. """ user = user_dic['login'] tz = user_dic['timezone'] offset = get_tz_offset(tz) if day_range == 0: date_from = '' date_to = '' else: date_from = (datetime.utcnow() - timedelta(days=day_range)).strftime('%Y-%m-%d %H:%M:%S') date_to = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') try: trend_list = db_get_otx_event_trend(user, pulse_id, date_from, date_to, offset) except Exception as err: api_log.error("[apimethod_get_otx_pulse_stats_event_trend] %s" % str(err)) return False, "Error retrieving the events from Pulses: %s" % str(err) return True, trend_list
def asynchronous_update(system_id, only_feed=False, update_key=""): """Launches an asynchronous update on the given system_ip Args: system_id (str): The system_id of the system to update. only_feed (boolean): A boolean to indicate that we need to update only the feed. Returns: (boolean, job_id): A tuple containing the result of the execution Examples: >>> asynchronous_update("11111111-1111-1111-111111111111") (True,"/var/log/alienvault/update/system_update.log") """ (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: error_msg = "[asynchronous_update] Error retrieving " + \ "the system ip for the system id " + \ "%s -> %s" % (system_ip, str(system_ip)) return False, error_msg job = alienvault_asynchronous_update.delay(system_ip, only_feed, update_key) if job is None: error_msg = "Cannot update system %s. " % system_id + \ "Please verify that the system is reachable." api_log.error(error_msg) return False, error_msg flush_cache(namespace="system_packages") return True, job.id
def ansible_download_release_info(system_ip): """Download release notes from alienvault.com Args: system_ip (str): ip of the host where we will download the release info file Returns: success (bool): True if successful, False otherwise msg (str): success/error message """ try: args = "url=http://data.alienvault.com/alienvault5/RELEASES/release_info " + \ "dest=/var/alienvault force=yes" response = ansible.run_module(host_list=[system_ip], module="get_url", use_sudo=True, args=args) (success, msg) = ansible_is_valid_response(system_ip, response) if success: msg = response['contacted'][system_ip]['msg'] except Exception as err: error_msg = "[ansible_download_release_info] An error occurred " + \ "while retrieving the release info <%s>" % str(err) api_log.error(error_msg) return False, error_msg return success, msg
def ossec_get_server_config(sensor_id): (success, system_ip) = get_sensor_ip_from_sensor_id(sensor_id) if not success: return False, "Invalid sensor id %s" % sensor_id success, ossec_directory = get_ossec_directory(sensor_id) if not success: api_log.error(str(ossec_directory)) return False, ossec_directory server_config_file = os.path.join(ossec_directory, OSSEC_CONFIG_SERVER_FILE_NAME) success, filename = fetch_file(system_ip=system_ip, src_file_path=OSSEC_CONFIG_SERVER_PATH, dst_file_path=server_config_file, fail_on_missing=True, flat=True) if not success: if str(filename).find('the remote file does not exist') > 0: if touch_file(server_config_file): filename = server_config_file else: api_log.error(str(filename)) return False, "Something wrong happened getting the HIDS server configuration file" success, result = set_ossec_file_permissions(server_config_file) if not success: return False, str(result) return True, filename
def apimethod_get_agentless_passlist(sensor_id): (success, system_ip) = get_sensor_ip_from_sensor_id(sensor_id) if not success: return False, "Invalid sensor id %s" % sensor_id success, base_path = get_base_path_from_sensor_id(sensor_id) if not success: return False, "Can't retrieve the destination path: %s" % base_path destination_path = base_path + "/ossec/agentless/" success, msg = create_local_directory(destination_path) if not success: api_log.error(str(msg)) return False, "Error creating directory '%s'" % destination_path dst_filename = destination_path+".passlist" success, msg = ans_ossec_get_agentless_passlist(system_ip=system_ip, destination_path=dst_filename) if not success: if str(msg).find('the remote file does not exist') > 0: if touch_file(dst_filename): success = True msg = dst_filename success, result = set_ossec_file_permissions(dst_filename) if not success: return False, str(result) return success, msg
def package_list(system_ip): """ Get a list with all the packages available Args: system_ip (str): String with a IP address. Returns: A tuple of the form (bool, data) where the first element is a true or false. The data parameter returns the error message on error or the dict with all the packages. Each package is a dic itself containing the version and the description of the package """ command = "aptitude -F '%p@@@%v@@@%d' --disable-columns search ~i" try: response = ansible.run_module( host_list=[system_ip], module="command", use_sudo="True", args=command) except Exception, exc: api_log.error( "Ansible Error: An error occurred while retrieving package list: %s" % str(exc)) return False, "Ansible Error: An error occurred while retrieving package list: %s" % str( exc)
def system_status(system_ip): """ Get the alienvault_status system info Args: system_ip (str): String with a IP address. Returns: A tuple of the form (bool, data) where the first element is a true or false. The data parameter returns the error message on error or the system status data if the exececution is correct. """ response = ansible.run_module( host_list=[system_ip], module="av_center_facts", args="sections=system_status") # Check if we have contacted the system if system_ip in response['contacted']: # Check the "failed" flag if not response['contacted'][system_ip].get('failed', False): result = APIResult(True, response['contacted'][system_ip]['data']) else: api_log.error( "Error calling module 'av_center_facts' msg:%s" % response['contacted'][system_ip].get('msg', "Unknown error")) result = APIResult( False, "Error calling module 'av_center_facts' msg:%s" % response['contacted'][system_ip].get('msg', "Unknown error")) else: api_log.error("Can't connect to system with IP %s msg:%s " % (system_ip, str(response['dark'][system_ip]))) result = APIResult( False, "Can't connect to system with IP %s msg:%s " % (system_ip, str(response['dark'][system_ip]))) return result
def get_linked_assets(): """ Get assets Returns: Dictionary with linked assets Raises: APICannotGetLinkedAssets """ assets_ids = {} query = "SELECT ha.host_id AS host_id, ha.sensor_id AS sensor_id, ha.agent_id FROM hids_agents ha WHERE ha.host_id is not NULL" try: asset_list = db.session.connection(mapper=Hids_Agents).execute(query) for asset in asset_list: ha_id = asset.agent_id ha_sensor_id = asset.sensor_id ha_host_id = asset.host_id ha_host_id = get_uuid_string_from_bytes(ha_host_id) assets_ids[ha_host_id] = { 'ha_id': ha_id, 'sensor_id': get_uuid_string_from_bytes(ha_sensor_id) } except Exception as msg: api_log.error("[get_linked_assets]: %s" % str(msg)) raise APICannotGetLinkedAssets() return assets_ids
def ansible_check_if_process_is_running(system_ip, ps_filter): """Check whether a process is running or not Args: system_ip(str): The system IP where we would like to run the ps filter ps_filter(str): Filter to grep the ps aux command Returns: (boolean,int): A tuple containing whether the operation was well or not and the number the process running that meet the filter """ try: rc = 0 cmd = 'ps aux | grep "%s" | grep -v grep | ' \ 'grep -v tail | wc -l' % re.escape(ps_filter) response = ansible.run_module(host_list=[system_ip], module="shell", use_sudo="True", args=cmd) (success, msg) = ansible_is_valid_response(system_ip, response) if not success: return False, msg rc = int(response['contacted'][system_ip]['stdout']) except Exception as exc: api_log.error("ansible_check_if_process_is_running: <%s>" % str(exc)) return False, 0 return success, rc
def ansible_pgrep(system_ip, pgrep_filter="''"): """ Launch a pgrep in system :system_ip: with filter :pgrep_filter: matched against all the command line (-f). Return a tuple list with (pid,command line for each filter) """ result = [] try: cmd = "/usr/bin/pgrep -a -f '%s'" % pgrep_filter response = ansible.run_module(host_list=[system_ip], module="shell", use_sudo=True, args=cmd) (success, msg) = ansible_is_valid_response(system_ip, response) if not success: api_log.error("[ansible_pgrep] Error: %s" % str(msg)) return False, str(msg) if response['contacted'][system_ip]['stdout'] != '': data = response['contacted'][system_ip]['stdout'].split("\n") else: data = [] result = [tuple(x.split(" ", 1)) for x in data] except Exception as exc: api_log.error("[ansible_pgrep] Error: %s" % str(exc)) return False, str(exc) return True, result
def ansible_get_child_alarms(system_ip, delay=1, delta=3): """ Get the alarms from remote system """ cmd = "echo \"select hex(event_id), timestamp, hex(backlog_id) FROM alarm WHERE status='closed' AND timestamp between DATE_SUB(utc_timestamp(), " \ "interval %u hour) AND DATE_SUB(utc_timestamp(), interval %u hour) UNION select hex(event_id), timestamp, hex(backlog_id) " \ "FROM alarm WHERE status='open' AND " \ "timestamp between DATE_SUB(utc_timestamp(), interval %u hour) AND DATE_SUB(utc_timestamp(), interval %u hour) ORDER BY timestamp DESC;\" | ossim-db " % ( delta + delay, delay, delta + delay, delay) api_log.debug("Query: %s" % cmd) response = ansible.run_module(host_list=[system_ip], module="shell", args=cmd) success, msg = ansible_is_valid_response(system_ip, response) if not success: return False, "[ansible_get_child_alarms] Can't retrieve remote alarms (%s) : %s" % (system_ip, msg) data = [] try: output = str(response['contacted'][system_ip]['stdout']) split = output.splitlines() # Discard first line if split: for line in split[1:]: # Omit header (event_id, timestamp, backlog_id) = line.split('\t') data.append(event_id) except KeyError: api_log.error("[ansible_get_child_alarms] Bad response from child server: %s" & str(output)) return False, "[ansible_get_child_alarms] Bad response from child server" return True, data
def reconfigure(system_ip): """ Runs an alienvault-reconfigure :param system_ip: The system IP where you want to run the alienvault-reconfig :return A tuple (success, error_message). """ rt = True error_str = "" try: command = """executable=/bin/bash alienvault-reconfig -c --center""" response = ansible.run_module(host_list=[system_ip], module="shell", args=command) if response['contacted'].has_key(system_ip): return_code = response['contacted'][system_ip]['rc'] error_str = response['contacted'][system_ip]['stderr'] if return_code != 0: rt = False else: rt = False error_str = response['dark'][system_ip]['msg'] except Exception, e: trace = traceback.format_exc() error_msg = "Ansible Error: An error occurred while running " + \ "alienvault-reconfig: %s \n trace: " % str(e) + \ "%s" % trace api_log.error(error_msg) rt = False
def connect_tunnel(system_ip, case_id): """ Connect to :system_ip: and enable the reverse tunnel with case :case_id: """ evars = {'ca_root': '/etc/ansible/playbooks/cacert.pem', 'remote_server': 'tractorbeam.alienvault.com', 'remote_port': '443', 'remote_user': '******', 'case_id': case_id, 'target': system_ip} response = ansible.run_playbook(playbook=PLAYBOOKS['ENABLE_TUNNEL'], host_list=[system_ip], extra_vars=evars, use_sudo=True) success, msg = ansible_is_valid_playbook_response(system_ip, response) if not success: # Log all the error to api_log # First api_log.error("ERROR: ansible.run_playbook " + msg) try: err = response['alienvault']['lasterror'] if type(err) == dict: return False, response['alienvault']['lasterror'][system_ip]['msg'] else: return False, msg except KeyError: return False, msg else: return True, ''
def get_host_id_by_ip_ctx(ip, ctx, output='str'): """ Returns an Asset ID given an IP address and context """ host_id = None try: if ip and ctx: ip_bin = get_ip_bin_from_str(ip) ctx_bin = get_bytes_from_uuid(ctx) query = db.session.query(Host.id).filter(Host.ctx == ctx_bin) host = query.join(Host_Ip, Host_Ip.host_id == Host.id).filter(Host_Ip.ip == ip_bin).one() host_id = host.id else: return False, "IP address and/or context could not be empty" except NoResultFound: return True, host_id except Exception as msg: msg = str(msg) api_log.error(msg) return False, "Asset ID not found in the system" if output == 'str': return True, get_uuid_string_from_bytes(host_id) else: return True, host_id
def ansible_get_update_info(system_ip): """Retrieves information about the system packages. Args: system_ip(str): IP of the system of which we want info Returns: success(Boolean), msg(str): A tuple containing the result of the query and the data """ try: response = ansible.run_module(host_list=[system_ip], module="av_update_info", use_sudo=True, args={}) (success, msg) = ansible_is_valid_response(system_ip, response) if success: msg = response['contacted'][system_ip]['data'] except Exception as err: error_msg = "[get_packages_info] An error occurred while " + \ "retrieving the system's package info <%s>" % str(err) api_log.error(error_msg) return False, error_msg return success, msg
def is_task_in_celery(task_id): """Look whether a task is scheduled, reserved or active Args: The task id Returns the task dictionary or None """ task = None try: i = inspect() task_list = {} active = i.active().copy() scheduled = i.scheduled().copy() reserved = i.reserved().copy() for node, tasks in active.iteritems(): for task in tasks: task_list[str(task['id'])] = task for node, tasks in reserved.iteritems(): for task in tasks: task_list[str(task['id'])] = task for node, tasks in scheduled.iteritems(): for task in tasks: task_list[str(task['id'])] = task if task_id in task_list: found = task_list[task_id].copy() return found except Exception as exp: api_log.error("[is_task_in_celery] An error occurred while reading the task list") return task
def apimethod_get_pulse_detail(pulse_id, hide_ioc=False): """Disable the config flag to start contributing to OTX Args: pulse_id(string): Pulse ID Returns: success (bool): True if successful, False elsewhere result(string): Error message if there was an error or empty string otherwise. """ pulse_id = pulse_id.lower() try: pulse_db = PulseDB() p_data = pulse_db.get(pulse_id) del pulse_db pulse = ast.literal_eval(p_data) pulse['total_indicators'] = len(pulse['indicators']) if hide_ioc is True: pulse.pop('indicators', False) elif len(pulse['indicators']) > 0: indicators = {} for ioc in pulse['indicators']: ioc_key = hashlib.md5(ioc.get('indicator', '')).hexdigest() indicators[ioc_key] = ioc pulse['indicators'] = indicators except RedisDBKeyNotFound, err: api_log.error("[apimethod_get_pulse_detail] Cannot find the Pulse ID [%s]: %s" % (str(pulse_id), str(err))) return False, "Cannot find the Pulse ID [%s]: %s" % (str(pulse_id), str(err))
def get_logfiles_for_host(system_ip, host_ips): """Returns a list of log files where the host ip have been found Args: system_ip(str): System IP host_ips(): list of hosts Returns: On success, it returns a hash table with the correct values, otherwise it returns an empty dict Note: DEPRECATED (Not used) """ logfiles = [] try: grep_filter = "|".join(ip for ip in host_ips) command = """executable=/bin/bash grep --exclude=\*.{tar.gz,dat,gz} -rIEo '%s' /var/log/ | sort -u """ % grep_filter response = ansible.run_module(host_list=[system_ip], module="shell", args=command) (success, msg) = ansible_is_valid_response(system_ip, response) if success: response = response['contacted'][system_ip]['stdout'].split('\n') for line in response: splitted_line = line.split(':') if len(splitted_line) != 2: continue log_filename = splitted_line[0] if log_filename not in logfiles: logfiles.append(log_filename) except Exception, e: api_log.error("get_logfiles_for_host: %s, r: %s" % (str(e), response))
def apimethod_monitor_nmap_scan(sensor_id, task_id): """Monitors an NMAP scan Args: sensor_id: The sensor id where the NMAP is working. task_id: The celery task id that is launching the NMAP Raises APICannotResolveSensorID APINMAPScanCannotRetrieveScanProgress """ (result, sensor_ip) = get_sensor_ip_from_sensor_id(sensor_id, local_loopback=False) if result is False: api_log.error( "[apimethod_monitor_nmap_scan] Cannot retrieve the sensor ip from the given sensor id <%s>" % sensor_id) raise APICannotResolveSensorID(sensor_id) try: nhosts = ansible_nmap_get_scan_progress(sensor_ip=sensor_ip, task_id=task_id) except Exception as e: api_log.error("[apimethod_monitor_nmap_scan] Cannot retrieve scan progress {0}".format(str(e))) raise APINMAPScanCannotRetrieveScanProgress() return nhosts
def update_latest_request(self, d_type, d_update=None): """Update the latest otx request timestamp Args: type (str): The type of date to update. update_date (str): Update Date Returns: Boolean """ date_type = self.date_types.get(d_type, None) if date_type is None: return False update_date = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%S") if d_update is None else d_update try: self.pulse_db.set_key_value(date_type, update_date) except Exception as err: api_log.error("Cannot save messages revision: %s" % str(err)) return False return True
def utc_to_local(utc_date, timezone): """Transform a utc date to a given timezone date: To convert from string to date: datetime.strptime('2015-05-26 15:00:00', '%Y-%m-%d %H:%M:%S') Args: utc_date(date) : Date in UTC format timezone(string) : Local Timezone (IE: "Europe/Madrid") Returns: date (date) : Date converted to local timezone """ try: local_tz = pytz.timezone(timezone) local_date = utc_date.replace(tzinfo=pytz.utc).astimezone(local_tz) except Exception as err: api_log.error("[utc_to_local] There was an error converting the date to local time: %s" % str(err)) # Saving the utc_date as local date to avoid further errors local_date = utc_date return local_date
def get_plugin_list(system_ip): """ Returns a hash table of plugin_name = plugin_id :param system_ip: System IP :return: On success, it returns a hash table with the correct values, otherwise it returns an empty dict """ pluginhash = {} try: command = """executable=/bin/bash grep "plugin_id=" /etc/ossim/agent/plugins/* | awk -F ":" '{print $1";"$2}'""" response = ansible.run_module(host_list=[system_ip], module="shell", args=command) response = response['contacted'][system_ip]['stdout'].split('\n') for line in response: plugin_name, plugin_id = line.split(';') not_used, pid = plugin_id.split('=') pluginhash[plugin_name] = pid except Exception, e: api_log.error("Ansible Error: get_plugin_list %s" % str(e))
def apimethod_get_nmap_scan(sensor_id, task_id): """Retrieves the result of an nmap scan Args: Raises: APINMAPScanCannotRetrieveBaseFolder APINMAPScanCannotCreateLocalFolder APINMAPScanReportNotFound APINMAPScanCannotReadReport """ directory = get_nmap_directory(sensor_id) nmap_report_path = "{0}/nmap_report_{1}.json".format(directory, task_id) if not os.path.isfile(nmap_report_path): raise APINMAPScanReportNotFound(nmap_report_path) try: with open(nmap_report_path, "r") as f: return json.loads(f.read()) except Exception as e: api_log.error("[apimethod_get_nmap_scan] {0}".format(str(e))) raise APINMAPScanCannotReadReport(nmap_report_path)
def get_remote_software_status(system_id): """Get the software status from a given AlienVault system or all systems The blueprint handle the following url: GET /av/api/1.0/system/<system_id>/status/software Args: system_id (str): String with system id (uuid) local or all """ no_cache = request.args.get('no_cache') if not is_json_boolean(no_cache): return make_error("Invalid value for the no_cache parameter", 500) no_cache = is_json_true(no_cache) success, result = apimethod_get_remote_software_update(system_id, no_cache) if not success: api_log.error("Error: " + str(result)) return make_error("Cannot retrieve packages status " + str(result), 500) return make_ok(**result)
def get_ossec_rule_filenames(sensor_ip): """ Get the ossec rule file names """ try: command = "/usr/bin/find /var/ossec/alienvault/rules/*.xml -type f -printf \"%f\n\"" response = _ansible.run_module(host_list=[sensor_ip], module="shell", args=command) if sensor_ip in response['dark'] or 'unreachable' in response: return False, make_err_message('[get_ossec_rule_filenames]', DEFAULT_ERR_MSG, str(response)) if 'failed' in response['contacted'][sensor_ip]: return False, make_err_message('[get_ossec_rule_filenames]', DEFAULT_ERR_MSG, str(response)) file_list = response['contacted'][sensor_ip]['stdout'].split('\n') except Exception, exc: api_log.error( "Ansible Error: An error occurred while running ossec_rule_filenames: %s" % str(exc)) return False, str(exc)
def ansible_get_log_lines(system_ip, logfile, lines): """Get a certain number of log lines from a given log file Args: system_ip (str): String with system ip. log_file (str): String with the name of the log file. lines (integer): Integer with the number of lines to display. """ command = "tail -%s %s | base64" % (str(lines), logfile) try: response = ansible.run_module(host_list=[system_ip], module="shell", use_sudo="True", args=command) except Exception, exc: error_msg = "Ansible Error: An error occurred retrieving " + \ "the log file: %s" % str(exc) api_log.error(error_msg) return False, error_msg
def save_plugin_from_raw_sql(plugin_sql): """ Stores plugin and related sids into the DB. Args: plugin_sql: (str) plugin sql statements. Returns: status: (bool) True if success or False otherwise. msg: (str) Error message in case of failure. """ status, msg = True, 'ok' try: db.session.begin() db.session.connection(mapper=Plugin).execute(text(plugin_sql)) db.session.commit() except Exception as e: api_log.error("[save_plugin_from_file] {}".format(str(e))) db.session.rollback() status, msg = False, 'Failed to save plugin into the DB.' return status, msg
def ossec_verify_agent_config_file(system_ip, path): try: command = "/var/ossec/bin/verify-agent-conf -f %s" % path response = _ansible.run_module(host_list=[system_ip], module="shell", args=command, use_sudo=True) if system_ip in response['dark'] or 'unreachable' in response: return False, "[ossec_verify_server_config_file] Something wrong happened while running ansible command %s" % str( response) if 'failed' in response['contacted'][system_ip]: return False, "[ossec_verify_server_config_file] Something wrong happened while running ansible command %s" % str( response) if response['contacted'][system_ip]['rc'] != 0: api_log.error(response['contacted'][system_ip]['stderr']) return False, response['contacted'][system_ip]['stderr'] except Exception, exc: api_log.error( "Ansible Error: An error occurred while running ossec_verify_server_config_file: %s" % str(exc)) return False, str(exc)
def db_system_update_hostname(system_id, hostname): try: sp_call = sqltext( "CALL system_update('%s','%s','','','','','','','','')" % (system_id, hostname)) db.session.begin() result = db.session.connection(mapper=System).execute(sp_call) data = result.fetchall() db.session.commit() if len(data) <= 0: return False, "Something wrong happened while updating system info in the database: %s" % str( data) if str(data[0]).find("updated") < 0 and str( data[0]).find("created") < 0: return False, "Something wrong happened while updating system info in the database: %s" % str( data[0]) except Exception, e: api_log.error(str(e)) db.session.rollback() return False, 'Something wrong happened while updating system info in the database'
def ansible_get_agent_config_yml(sensor_ip): """Get config.yml file and parse it""" config_file = '/etc/ossim/agent/config.yml' local_file = '/var/tmp/{0}{1}'.format(sensor_ip, config_file) device_list = {} try: success, dst = fetch_file(sensor_ip, config_file, '/var/tmp') except Exception as exc: api_log.error("[ansible_get_agent_config_yml] Error: %s" % str(exc)) return False, str(exc) if not os.path.exists(local_file): api_log.info( "[ansible_get_agent_config_yml] File {0} not found in {1}".format( config_file, local_file)) else: try: with open(local_file, 'r') as f: content = yaml.load(f.read()) if "plugins" in content: for plg in content['plugins']: for path, info in plg.iteritems(): if "DEFAULT" in info: data = info['DEFAULT'] device_list[data['device_id']] = [ ] # Support more than one plugin per asset for plg in content['plugins']: for path, info in plg.iteritems(): if "DEFAULT" in info: data = info['DEFAULT'] device_list[data['device_id']].append( data['pid'] ) # Support more than one plugin per asset os.remove(local_file) except Exception as exc: api_log.error( "[ansible_get_agent_config_yml] Unable to parse yml: %s" % str(exc)) return False, str(exc) return True, device_list
def update_hids_agent_status(agent_id, sensor_id, agent_status): """ Update status of HIDS agent Raises: APICannotResolveSensorID APIInvalidHIDSAgentID APICannotUpdateHIDSAgent """ if sensor_id is None: api_log.error( "[update_hids_agent_status]: Sensor ID could not be empty") raise APICannotResolveSensorID(sensor_id) if agent_id is None: api_log.error( "[update_hids_agent_status]: Agent ID could not be empty") raise APIInvalidHIDSAgentID(agent_id) try: sensor_id_bin = get_bytes_from_uuid(sensor_id) status_integer = Hids_Agents.get_status_integer_from_string( agent_status) db.session.begin() db.session.query(Hids_Agents).filter( and_(Hids_Agents.agent_id == agent_id, Hids_Agents.sensor_id == sensor_id_bin)).update( {"agent_status": status_integer}) db.session.commit() except Exception as msg: db.session.rollback() api_log.error("[update_hids_agent_status]: %s" % str(msg)) raise APICannotUpdateHIDSAgent(agent_id, sensor_id)
def delete_hids_agent(agent_id, sensor_id): """ Delete a HIDS agent Args: agent_id(str): HIDS agent ID sensor_id(str): Sensor ID Raises: APICannotResolveSensorID APIInvalidHIDSAgentID APICannotDeleteHIDSAgent """ if sensor_id is None: api_log.error("[delete_hids_agent]: Sensor ID could not be empty") raise APICannotResolveSensorID(sensor_id) if agent_id is None: api_log.error("[delete_hids_agent]: Agent ID could not be empty") raise APIInvalidHIDSAgentID(agent_id) try: sensor_id_bin = get_bytes_from_uuid(sensor_id) db.session.begin() db.session.query(Hids_Agents).filter( and_(Hids_Agents.agent_id == agent_id, Hids_Agents.sensor_id == sensor_id_bin)).delete() db.session.commit() except Exception as msg: db.session.rollback() api_log.error("[delete_hids_agent] %s" % str(msg)) raise APICannotDeleteHIDSAgent(agent_id, sensor_id)
def get_service_status_by_ip(system_ip): """Retrieves the processes status for suricata, prads, and ossec Args: system_ip(str): System IP Returns: It returns a hastable with the services status """ try: processes = { 'AlienVault_NIDS': 'down', 'AlienVault_HIDS': 'down', 'prads': 'down' } command = "ps ax | egrep 'suricata|prads|ossec' | awk '{print $5}' | grep -v egrep" response = _ansible.run_module(host_list=[system_ip], module="shell", args=command) result, msg = ansible_is_valid_response(system_ip, response) if not result: return False, msg lines = response['contacted'][system_ip]['stdout'].split("\n") for line in lines: if re.search('suricata', line) is not None: processes['AlienVault_NIDS'] = 'up' elif re.search('ossec', line) is not None: processes['AlienVault_HIDS'] = 'up' elif re.search('prads', line) is not None: processes['prads'] = 'up' except Exception, msg: api_log.error( "Ansible Error: An error occurred while retrieving processes status for sensor %s: %s" % (str(system_ip), str(msg))) return False, str(msg)
def ansible_run_nmap_scan(sensor_ip, target, scan_type, rdns, scan_timing, autodetect, scan_ports, job_id): """Runs a nmap scan on the given sensor and with the given parameters. Args: sensor_ip: The system IP where you want to get the [sensor]/interfaces from ossim_setup.conf target: IP address of the component where the NMAP will be executed scan_type: Sets the NMAP scan type rdns: Tells Nmap to do reverse DNS resolution on the active IP addresses it finds scan_timing: Set the timing template autodetect: Aggressive scan options (enable OS detection) scan_ports: Scan only specified ports Returns: A tuple (success|error, data | msgerror) """ args = "target=%s" % target if scan_type is not None: args += " scan_type=%s" % scan_type if rdns is not None: args += " rdns=%s" % str(rdns).lower() if scan_timing is not None: args += " scan_timming=%s" % scan_timing if autodetect is not None: args += " autodetect=%s" % str(autodetect).lower() if scan_ports is not None: args += " scan_ports=%s" % scan_ports args += " job_id={0}".format(job_id) try: response = ansible.run_module([sensor_ip], 'av_nmap', args) (success, msg) = ansible_is_valid_response(sensor_ip, response) if not success: api_log.error("[ansible_run_nmap_scan] Error: %s" % str(msg)) return False, str(msg) data = "" if response['contacted'][sensor_ip]['data'] != '': data = response['contacted'][sensor_ip]['data'] except Exception as exc: api_log.error("[ansible_run_nmap_scan] Error: %s" % str(exc)) return False, str(exc) return True, data
def ossec_get_agent_config(sensor_id): (success, system_ip) = get_sensor_ip_from_sensor_id(sensor_id) if not success: return False, "Invalid sensor id %s" % sensor_id success, ossec_directory = get_ossec_directory(sensor_id) if not success: api_log.error(str(ossec_directory)) return False, ossec_directory agent_config_file = os.path.join(ossec_directory, OSSEC_CONFIG_AGENT_FILE_NAME) success, filename = fetch_file(system_ip=system_ip, src_file_path=OSSEC_CONFIG_AGENT_PATH, dst_file_path=agent_config_file, fail_on_missing=True, flat=True) try: if not success: if str(filename).find('the remote file does not exist') > 0: if touch_file(agent_config_file): success = True filename = agent_config_file except Exception as err: import traceback api_log.error("EX: %s, %s" % (str(err), traceback.format_exc())) if not success: api_log.error(str(filename)) return False, "Something wrong happened getting the HIDS agent configuration file" success, result = set_ossec_file_permissions(agent_config_file) if not success: return False, str(result) return True, filename
def ossec_add_agentless(system_ip, host=None, user=None, password=None, supassword=None): """ Add a agentless monitoring system @param system_ip Sensor IP where we're going to modify the ossec configuration @param host we're going to add @param user user we use to connect to host @param password password for user @param supassword optional password use. """ if not (host and user and password): api_log.error( "[ossec_add_agentless] Missing mandatory parameter: Host, user or password (%s, %s, %s)" % (host, user, password)) return ( False, "[ossec_add_agentless] Missing mandatory parameter: Host, user or password (%s, %s, %s)" % (host, user, password)) try: command = "/var/ossec/agentless/register_host.sh add %s@%s %s %s" % ( user, host, password, supassword if supassword != None else '') response = _ansible.run_module(host_list=[system_ip], module="shell", args=command, use_sudo=True) result, msg = ansible_is_valid_response(system_ip, response) if not result: return False, msg script_return_code = int(response['contacted'][system_ip]['rc']) script_output = response['contacted'][system_ip]['stdout'] if script_return_code != 0: return False, "[ossec_add_agentless] Something wrong happened while running ansible command ->'%s'" % str( response) return True, script_output except Exception as err: return False, "[ossec_control] Something wrong happened while running ansible command -> '%s'" % str( err)
def connect(self, attempts=3, wait_time=10): """Connects to the server and starts the handshake""" try: tries = 0 connected = False self.close() while tries < attempts and connected is False: if tries > 0: time.sleep(wait_time) tries += 1 api_log.debug("IDM connector - Tries: %s connected: %s" % (tries, connected)) api_log.debug("IDM connector - Conn: %s" % self.conn) if self.conn is not None: self.conn.close() self.conn = None self.conn = Connection(ip=self.ip, port=self.port) api_log.debug( "IDM connector - Creating a new Connection object") if not self.conn.connect(attempts=attempts, default_wait=wait_time): continue # Send the connection message: self.sequence_id += 1 if self.sensor_id is None: connect_message = self.CONNECT_MESSAGE_2.format( self.sequence_id) else: connect_message = self.CONNECT_MESSAGE.format( self.sequence_id, self.sensor_id) api_log.error( "IDM connector - Send: {0}".format(connect_message)) if not self.send(connect_message): api_log.error( "IDM connector - Cannot send the connection message") self.conn.close() continue connection_message_response = self.recv() if not self.__process_connection_message_response( connection_message_response): api_log.error( "IDM connector - Cannot connect to the server, invalid response" ) self.conn.close() continue connected = True if not connected: return False except Exception as err: api_log.debug( "IDM connector - Cannot connect to the server.... %s" % str(err)) self.close() return False return True
def apimethod_run_nmap_scan(sensor_id, target, idm, scan_type, rdns, scan_timing, autodetect, scan_ports, output_file_prefix="", save_to_file=False, job_id=""): """Launches an MAP scan Args: sensor_id: The system IP where you want to get the [sensor]/interfaces from ossim_setup.conf target: IP address of the component where the NMAP will be executed idm: Convert results into idm events scan_type: Sets the NMAP scan type rdns: Tells Nmap to do reverse DNS resolution on the active IP addresses it finds scan_timing: Set the timing template autodetect: Aggressive scan options (enable OS detection) scan_ports: Only scan specified ports output_file_prefix: Prefix string to be added to the output filename save_to_file: Indicates whether you want to save the NMAP report to a file or not. job_id: Celery job ID. Returns: nmap_report: The NMAP report or the filename where the report has been saved. Raises: APINMAPScanCannotRun APICannotResolveSensorID APINMAPScanCannotRetrieveBaseFolder APINMAPScanCannotCreateLocalFolder """ (result, sensor_ip) = get_sensor_ip_from_sensor_id(sensor_id, local_loopback=False) if result is False: api_log.error( "[apimethod_run_nmap_scan] Cannot retrieve the sensor ip from the given sensor id <%s>" % sensor_id) raise APICannotResolveSensorID(sensor_id) success, nmap_report = ansible_run_nmap_scan(sensor_ip=sensor_ip, target=target, scan_type=scan_type, rdns=rdns, scan_timing=scan_timing, autodetect=autodetect, scan_ports=scan_ports, job_id=job_id) if not success: api_log.error('Failed to launch NMAP scan: %s' % nmap_report) raise APINMAPScanCannotRun(nmap_report) filename = None if save_to_file: base_path = get_nmap_directory(sensor_id) filename = "%s/nmap_report_%s.json" % (base_path, output_file_prefix) with open(filename, "w") as f: f.write(json.dumps(nmap_report)) if idm: conn = IDMConnection(sensor_id=sensor_id) if conn.connect(): conn.send_events_from_hosts(nmap_report) try: if filename is not None: os.remove(filename) except Exception: pass else: api_log.error("[apimethod_run_nmap_scan] Cannot connect with the IDM Service") try: apimethods_nmap_purge_scan_files(job_id) except Exception as exp: api_log.warning("[apimethod_run_nmap_scan] Cannot purge the scan files %s" % str(exp)) return nmap_report
def reload_hosts(self): """Builds an IDM message to reload the hosts""" try: self.sequence_id += 1 message = 'reload-hosts id="' + str(self.sequence_id) + '"\n' api_log.info("Sending the reload host message") self.send(message) connection_message_response = self.recv() if not self.__process_connection_message_response( connection_message_response): api_log.error( "Server connector - Cannot connect to the server, invalid response" ) except Exception as e: api_log.debug( "Server connector, cannot send the reload host message")
def remove_pulses(self, pulses): """Delete a list of pulses from redis. Args: pulses(list): List of pulse IDs we want to remove Returns: integer: Number of pulses removed. """ del_pulses = len(pulses) if del_pulses > 0: for p_id in pulses: try: pulse = ast.literal_eval(self.pulse_db.get(p_id)) self.pulse_db.delete_key(p_id) self.pulse_correlation_db.delete_pulse(pulse) except RedisDBKeyNotFound: del_pulses -= 1 continue except Exception as err: api_log.error("Error deleting Pulse: {}".format(err)) del_pulses -= 1 continue return del_pulses
def is_task_in_rabbit(task_id): """Checks if the task is in rabbit or not. If the task is found returns the json data, otherwise returns none {u'retries': 0, u'task': u'celerymethods.tasks.monitor_tasks.monitor_retrieves_remote_info', u'eta': None, u'args': [], u'expires': None, u'callbacks': None, u'errbacks': None, u'kwargs': {}, u'id': u'de1cd3b1-d001-4bea-a050-8ffe610bee21', u'utc': False} """ try: conn = amqp.Connection(host="localhost:5672 ", userid="guest", password="******", virtual_host="/", insist=False) chan = conn.channel() while True: msg = chan.basic_get(queue="celery") if msg is not None: try: task_json = json.loads(msg.body) if task_json['id'] == task_id: api_log.warning("Task found in rabbit... that means celery is busy..") return task_json except Exception as cannot_parse: api_log.warning("Cannot parse rabbit message: %s" % str(cannot_parse)) else: break except Exception as e: api_log.error("Cannot inspect rabbitmq: %s" % str(e)) return None
def get_hosts_in_syslog(system_ip): """Returns a hash table of host_ip = log filename found under the folder /var/log/* Args: system_ip (str): System IP Returns: On success, it returns a hash table with the correct values, otherwise it returns an empty dict Note: DEPRECATED (Not used) 4 """ hostlist = {} try: #command = """executable=/bin/bash grep -rEo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' /var/log/* | sort -u """ command = """grep --exclude=\*.{tar.gz,dat,gz} -rIEo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' /var/log/* | sort -u """ response = ansible.run_module(host_list=[system_ip], module="shell", args=command) (success, msg) = ansible_is_valid_response(system_ip, response) if success: response = response['contacted'][system_ip]['stdout'].split('\n') for line in response: split_line = line.split(':') if len(split_line) == 2: log_filename = split_line[0] host_ip = split_line[1] else: continue if not hostlist.has_key(host_ip): hostlist[host_ip] = [] if log_filename not in hostlist[host_ip]: hostlist[host_ip].append(log_filename) except Exception, e: api_log.error("get_hosts_in_syslog error: %s, %s" % (str(e), traceback.format_exc()))
def get_ossec_rule_filenames(sensor_ip): """ Get the ossec rule filenames """ try: command = "/usr/bin/find /var/ossec/rules/*.xml -type f -printf \"%f\n\"" response = _ansible.run_module(host_list=[sensor_ip], module="shell", args=command) if sensor_ip in response['dark'] or 'unreachable' in response: return False, "[get_ossec_rule_filenames] Something wrong happened while running ansible command %s" % str( response) if 'failed' in response['contacted'][sensor_ip]: return False, "[get_ossec_rule_filenames] Something wrong happened while running ansible command %s" % str( response) file_list = response['contacted'][sensor_ip]['stdout'] file_list = file_list.split('\n') except Exception, exc: api_log.error( "Ansible Error: An error occurred while running ossec_rule_filenames: %s" % str(exc)) return False, str(exc)
def cache_func(*args, **kwargs): cache = _get_cache(namespace=namespace, expire=expire) # Handle cases where caching is down or otherwise not available. if cache.connection is None: return function(*args, **kwargs) # Key will be either a md5 hash or just pickle object, # in the form of `function name`:`key` if cache.hashkeys: key = hashlib.md5(json.dumps(args)).hexdigest() else: key = json.dumps(args) cache_key = '%s:%s' % (function.__name__, key) if 'no_cache' not in kwargs or not kwargs['no_cache']: try: return cache.get_json(cache_key) except (ExpiredKeyException, CacheMissException): pass except Exception, msg: api_log.error(str(msg))
def insert_plugin_data(plugin_id, plugin_name, vendor, model, version, ctx="", nsids=0, plugin_type=1, product_type=None): """ Inserts custom plugin data into the DB. Args: plugin_id: (int) unique plugin ID. plugin_name: (str) plugin name. vendor: (str) vendor name. model: (str) model name. version: (str) plugin version. ctx: (str) plugin_type: (int) plugin type: 1 - custom plugin. product_type: (int) ID of product data source. Returns: tuple with two elements: (bool) True on success or False otherwise; error message in case of fail """ try: db.session.begin() plugin_data = PluginData() plugin_data.ctx = ctx plugin_data.plugin_id = plugin_id plugin_data.plugin_name = plugin_name plugin_data.vendor = vendor plugin_data.model = model plugin_data.version = version plugin_data.nsids = nsids plugin_data.plugin_type = plugin_type plugin_data.product_type = product_type db.session.add(plugin_data) db.session.commit() except Exception as err: api_log.error("[insert_plugin_data] Failed due to: {}".format(str(err))) db.session.rollback() return False, 'Failed to insert plugin data for "{}-{}": {}'.format(plugin_id, plugin_name, str(err)) return True, ""
def ansible_nfsen_reconfigure(system_ip): """Runs a nfsen reconfigure Args: system_ip(str): The system IP where we would like to run the command Returns: (boolean,int): A tuple containing whether the operation was well or not """ try: cmd = '/usr/share/ossim/scripts/nfsen_reconfig.sh' response = ansible.run_module(host_list=[system_ip], module="shell", use_sudo="True", args=cmd) (success, msg) = ansible_is_valid_response(system_ip, response) rc = int(response['contacted'][system_ip]['rc']) if rc != 0: success = False rc = response['contacted'][system_ip]['stderr'] except Exception as exc: api_log.error("ansible_nfsen_reconfigure <%s>" % str(exc)) return False, 0 return success, rc
def get_system_status(system_id): """ Handle the url GET /av/api/1.0/system/<system_id>/status/general?no_cache=<boolean> Args: system_id (str): String with system id (uuid) or local """ no_cache = True if request.args.get('no_cache', 'false') == 'true' else False success, result = system_all_info(system_id, no_cache=no_cache) if not success: api_log.error( "Cannot retrieve system status for system_id %s. Error: %s" % (system_id, str(result))) api_log.error( "Failed API call: remote addr = %s, host addr = %s, blueprint = %s, URL = %s" % (request.remote_addr, request.host, request.blueprint, request.base_url)) return make_error( "Cannot retrieve system status for system %s" % system_id, 500) return make_ok(**result)
def get_tz_offset(tz): """Transform a timezone string to timezone offset: Example: "Europe/Madrid" --> "+02:00" Args: tz(string): Timezone string ('Europe/Madrid') Returns: format_offset (string): Offset Timezone ('+03:00') """ try: offset = datetime.now(pytz.timezone(tz)).strftime('%z') regexp = re.compile('(\+|\-)(\d\d)(\d\d)') matcher = regexp.search(offset) format_offset = "%s%s:%s" % (matcher.group(1), matcher.group(2), matcher.group(3)) except Exception as err: api_log.error( "[get_tz_offset] There was an error getting the timezone offset: %s" % str(err)) format_offset = '+00:00' return format_offset