def get_support_info(system_id, ticket): args = {} args["output_type"] = "support" args["output_raw"] = "True" args["output_file_prefix"] = ticket (success, ip) = get_system_ip_from_system_id(system_id) if not success: return (False, "Bad system_id '%s'" % system_id) if not ticket.isdigit() or len(ticket) != 8: return (False, "Bad ticket id format: %s" % ticket) file_uploaded = False file_name = "" data = get_doctor_data([ip], args) if ip in data["dark"]: return (False, data["dark"][ip]["msg"]) if data["contacted"][ip]["rc"] == 0: file_uploaded = True elif data["contacted"][ip]["rc"] == 1: file_name = data["contacted"][ip]["data"].replace("\n", "") else: return (False, "Error Calling support tool") if not file_uploaded: (success, data) = fetch_file(ip, file_name, file_name) return (True, {"file_uploaded": file_uploaded, "file_name": file_name})
def get_system_config_general(system_id, no_cache=False): (success, system_ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret (success, config_values) = get_av_config(system_ip, {'general_admin_dns': '', 'general_admin_gateway': '', 'general_admin_ip': '', 'general_admin_netmask': '', 'general_hostname': '', 'general_interface': '', 'general_mailserver_relay': '', 'general_mailserver_relay_passwd': '', 'general_mailserver_relay_port': '', 'general_mailserver_relay_user': '', 'general_ntp_server': '', 'general_profile': '', 'firewall_active': '', 'update_update_proxy': '', 'update_update_proxy_dns': '', 'update_update_proxy_pass': '', 'update_update_proxy_port': '', 'update_update_proxy_user': '', 'vpn_vpn_infraestructure': '' }) if not success: api_log.error("system: get_config_general error: " + str(config_values)) return (False, "Cannot get general configuration info %s" % str(config_values)) return (True, config_values)
def get_system_config_general(system_id, no_cache=False): (success, system_ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret (success, config_values) = get_av_config( system_ip, { 'general_admin_dns': '', 'general_admin_gateway': '', 'general_admin_ip': '', 'general_admin_netmask': '', 'general_hostname': '', 'general_interface': '', 'general_mailserver_relay': '', 'general_mailserver_relay_passwd': '', 'general_mailserver_relay_port': '', 'general_mailserver_relay_user': '', 'general_ntp_server': '', 'general_profile': '', 'firewall_active': '', 'update_update_proxy': '', 'update_update_proxy_dns': '', 'update_update_proxy_pass': '', 'update_update_proxy_port': '', 'update_update_proxy_user': '' }) if not success: api_log.error("system: get_config_general error: " + str(config_values)) return (False, "Cannot get general configuration info %s" % str(config_values)) return (True, config_values)
def apimethod_check_asynchronous_command_return_code(system_id, rc_file): """Check the return code of a previously asynchronous request Args: system_ip (str): The system_id where you want to know if the process is running rc_file(str): The return code file Returns: (boolean, str): A tuple containing the result of the execution Examples: apimethod_ansible_check_asynchronous_command_return_code("11111111-1111-1111-1111-1111222244445555", "/var/log/alienvault/update/system_reconfigure.log.rc") (True,0) """ (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: error_msg = "[apimethod_ansible_check_" + \ "asynchronous_command_return_code] " + \ "Error retrieving the system ip " + \ "for the system id %s -> %s" % (system_ip, str(system_ip)) return success, error_msg return ansible_check_asynchronous_command_return_code(system_ip, rc_file)
def apimethod_get_update_info(system_id, no_cache=False): """Retrieves the system update information Args: system_id(str): The system id of which we want to know if it has available updates Returns: (success,data): success=True when the operation when ok, otherwise success=False. On success data will contain a json object with the updates information. """ try: (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: error_msg = "[apimethod_get_packages_info] Error retrieving " + \ "the system ip for the system id " + \ "%s -> %s" % (system_ip, str(system_ip)) return success, error_msg success, data = ansible_get_update_info(system_ip) except Exception as err: error_msg = "[apimethod_get_packages_info] " + \ "An error occurred while retrieving " + \ "the update info <%s>" % str(err) return False, error_msg return success, data
def asynchronous_update(system_id, only_feed=False, update_key=""): """Launches an asynchronous update on the given system_ip Args: system_id (str): The system_id of the system to update. only_feed (boolean): A boolean to indicate that we need to update only the feed. Returns: (boolean, job_id): A tuple containing the result of the execution Examples: >>> asynchronous_update("11111111-1111-1111-111111111111") (True,"/var/log/alienvault/update/system_update.log") """ (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: error_msg = "[asynchronous_update] Error retrieving " + \ "the system ip for the system id " + \ "%s -> %s" % (system_ip, str(system_ip)) return False, error_msg job = alienvault_asynchronous_update.delay(system_ip, only_feed, update_key) if job is None: error_msg = "Cannot update system %s. " % system_id + \ "Please verify that the system is reachable." api_log.error(error_msg) return False, error_msg flush_cache(namespace="system_packages") return True, job.id
def get_support_info (system_id, ticket): args = {} args['output_type'] = 'support' args['output_raw'] = 'True' args['output_file_prefix'] = ticket (success, ip) = get_system_ip_from_system_id(system_id) if not success: return (False, "Bad system_id '%s'" % system_id) if not ticket.isdigit() or len(ticket) != 8: return (False, "Bad ticket id format: %s" % ticket) file_uploaded = False file_name = '' data = get_doctor_data ([ip], args) if ip in data['dark']: return (False, data['dark'][ip]['msg']) if data['contacted'][ip]['rc'] == 0: file_uploaded = True elif data['contacted'][ip]['rc'] == 1: file_name = data['contacted'][ip]['data'].replace('\n', '') else: return (False, "Error Calling support tool") if not file_uploaded: (success, data) = fetch_file(ip, file_name, file_name) return (True, {'file_uploaded': file_uploaded, 'file_name': file_name})
def asynchronous_update(system_id, only_feed=False, update_key=""): """Launches an asynchronous update on the given system_ip Args: system_id (str): The system_id of the system to update. only_feed (boolean): A boolean to indicate that we need to update only the feed. Returns: (boolean, job_id): A tuple containing the result of the execution Examples: >>> asynchronous_update("11111111-1111-1111-111111111111") (True,"/var/log/alienvault/update/system_update.log") """ (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: error_msg = "[asynchronous_update] Error retrieving " + \ "the system ip for the system id " + \ "%s -> %s" % (system_ip, str(system_ip)) return False, error_msg job = alienvault_asynchronous_update.delay(system_ip, only_feed, update_key) if job is None: error_msg = "Cannot update system %s. " % system_id + \ "Please verify that the system is reachable." api_log.error(error_msg) return False, error_msg flush_cache(namespace="system_packages") return True, job.id
def get_system_config_alienvault(system_id, no_cache=False): (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: return (False, system_ip) (success, config_values) = get_av_config( system_ip, { 'framework_framework_ip': '', 'sensor_detectors': '', 'sensor_interfaces': '', 'sensor_mservers': '', 'sensor_netflow': '', 'sensor_networks': '', 'server_server_ip': '', 'server_alienvault_ip_reputation': '', 'ha_ha_virtual_ip': '', 'ha_ha_role': '', }) if not success: api_log.error("system: get_config_alienvault error: " + str(config_values)) return (False, "Cannot get AlienVault configuration info %s" % str(config_values)) return (True, config_values)
def get_plugin_sids_package(system_id, md5): """ Check the :system_id: system if its alienvault-plugin-sids package has md5 sum of :md5:. Download the package from remote system. check if not reconfig / update is running. Install package """ # First, check remote md5 rt = False emsg = '' try: result, info = get_plugin_package_info_from_system_id(system_id) if not result: raise Exception("Can't obtain alienvault-plugin-sid info for system %s : %s" % (system_id, str(info))) if info['md5'] != md5: raise Exception("md5 provided doesn't match with stored md5") # Use ansible to download file to temp directory result, ipremote = get_system_ip_from_system_id(system_id) if not result: raise Exception("Can't obtain remote system ip") result, iplocal = get_system_ip_from_local() if not result: raise Exception("Can't obtain local system ip") result, idlocal = get_system_id_from_local() if not result: raise Exception("Can't obtain local system id") # Create a temp file temp = NamedTemporaryFile(delete=True) tempname = temp.name plugin_package = "alienvault-plugin-sids_" + info['version'] + "_all.deb" remote_path = "/var/cache/apt/archives" result, emsg = fetch_if_changed(ipremote, os.path.join(remote_path, plugin_package), iplocal, tempname) if not result: raise Exception("Can't copy remote from %s file name %s Error: %s" % (ipremote, os.path.join(remote_path, plugin_package), emsg)) shutil.copy(tempname, remote_path) # Atomic rename os.rename(os.path.join(remote_path, os.path.basename(tempname)), os.path.join(remote_path, plugin_package)) # Check if we're not updaing / configuring result, status = check_update_and_reconfig_status(idlocal) if not result: raise Exception("Can't check current status reconfig / update") if status['alienvault-update']['job_status'] == 'running': raise Exception("alienvault-update running") if status['alienvault-reconfig']['job_status'] == 'running': raise Exception("alienvault-reconfig running") if status['ossim-reconfig']['job_status'] == 'running': raise Exception("ossim-reconfig running") # Okey, install package result, status = install_debian_package([iplocal], os.path.join(remote_path, plugin_package)) if not result: raise Exception("Can't install %s" % os.path.join(remote_path, plugin_package)) rt = True emsg = '' except Exception as excep: emsg = str(excep) rt = False return (rt, emsg)
def get_support_info (system_id, ticket): args = {} args['output_type'] = 'support' args['output_raw'] = 'True' args['verbose'] = 2 args['output_file_prefix'] = ticket (success, ip) = get_system_ip_from_system_id(system_id) if not success: return (False, "Bad system_id '%s'" % system_id) if not ticket.isdigit() or len(ticket) != 8: return (False, "Bad ticket id format: %s" % ticket) file_uploaded = False file_name = '' data = get_doctor_data ([ip], args) if ip in data['dark']: return (False, data['dark'][ip]['msg']) if data['contacted'][ip]['rc'] == 0: file_uploaded = True elif data['contacted'][ip]['rc'] == 1: file_name = data['contacted'][ip]['data'].replace('\n', '') # Clean to extract the filename file_name = re.sub(r'.*\/var\/ossim', '/var/ossim', file_name) file_name = re.sub(r'\.doctor.*', '.doctor', file_name) else: return (False, "Error Calling support tool") if not file_uploaded: (success, data) = fetch_file(ip, file_name, file_name) return (True, {'file_uploaded': file_uploaded, 'file_name': file_name})
def set_config_alienvault(system_id): param_names = [ 'framework_framework_ip', 'sensor_detectors', 'sensor_interfaces', 'sensor_mservers', 'sensor_networks', 'server_server_ip' ] (success, system_ip) = ret = get_system_ip_from_system_id(system_id) if not success: return make_error(system_ip, 500) set_values = {} for key, value in request.args.iteritems(): if key not in param_names: return make_error("Bad param %s" % key, 400) else: set_values[key] = value (success, config_values) = set_av_config(system_ip, set_values) if not success: current_app.logger.error("system: set_config_alienvault error: " + str(config_values)) return make_error( "Cannot set AlienVault configuration info %s" % str(config_values), 500) flush_cache(namespace="system") job = alienvault_asynchronous_reconfigure.delay(system_id) return make_ok(job_id=job.id)
def get_system_config_general(system_id, no_cache=False): (success, system_ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret (success, config_values) = get_av_config( system_ip, { "general_admin_dns": "", "general_admin_gateway": "", "general_admin_ip": "", "general_admin_netmask": "", "general_hostname": "", "general_interface": "", "general_mailserver_relay": "", "general_mailserver_relay_passwd": "", "general_mailserver_relay_port": "", "general_mailserver_relay_user": "", "general_ntp_server": "", "general_profile": "", "firewall_active": "", "update_update_proxy": "", "update_update_proxy_dns": "", "update_update_proxy_pass": "", "update_update_proxy_port": "", "update_update_proxy_user": "", }, ) if not success: api_log.error("system: get_config_general error: " + str(config_values)) return (False, "Cannot get general configuration info %s" % str(config_values)) return (True, config_values)
def get_system_config_alienvault(system_id, no_cache=False): (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: return (False, system_ip) (success, config_values) = get_av_config( system_ip, { "framework_framework_ip": "", "sensor_detectors": "", "sensor_interfaces": "", "sensor_mservers": "", "sensor_netflow": "", "sensor_networks": "", "server_server_ip": "", "server_alienvault_ip_reputation": "", "ha_ha_virtual_ip": "", "ha_ha_role": "", }, ) if not success: api_log.error("system: get_config_alienvault error: " + str(config_values)) return (False, "Cannot get AlienVault configuration info %s" % str(config_values)) return (True, config_values)
def apimethod_get_update_info(system_id, no_cache=False): """Retrieves the system update information Args: system_id(str): The system id of which we want to know if it has available updates Returns: (success,data): success=True when the operation when ok, otherwise success=False. On success data will contain a json object with the updates information. """ try: (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: error_msg = "[apimethod_get_packages_info] Error retrieving " + \ "the system ip for the system id " + \ "%s -> %s" % (system_ip, str(system_ip)) return success, error_msg success, data = ansible_get_update_info(system_ip) except Exception as err: error_msg = "[apimethod_get_packages_info] " + \ "An error occurred while retrieving " + \ "the update info <%s>" % str(err) return False, error_msg return success, data
def restore_backup(system_id='local', backup_type='configuration', backup_name='', backup_pass=''): """ Restore backup in the system """ success, system_ip = get_system_ip_from_system_id(system_id) if not success: api_log.error(str(system_ip)) error_msg = "Error retrieving the system ip for the system id %s -> %s" % (system_id, str(system_ip)) return False, error_msg backup_name = os.path.basename(backup_name) success, backup_path = secure_path_join(BACKUP_PATH, backup_name) if not success: api_log.error("restore backup: %s '%s'" % (backup_path, backup_name)) return False, "" try: success, msg = run_restore(target=system_ip, backup_type=backup_type, backup_file=backup_path, backup_pass=backup_pass) if not success: api_log.error("restore_backup: %s" % msg) error_msg = "Error trying to restore the backup '%s': %s" % (backup_name, msg) return False, error_msg except Exception as e: api_log.info("restore_backup Error: %s" % str(e)) error_msg = "Error trying to restore the backup '%s': %s" % (backup_name, str(e)) return False, error_msg return success, msg
def dns_resolution(system_id): """ Check the DNS name resolution. """ using_proxy = False dns_lookup = 'data.alienvault.com' (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: return (False, "Error translating system id to ip") (success, data) = get_av_config(system_ip, {'update_update_proxy': ''}) if not success: return (False, "Error getting proxy configuration") if 'update_update_proxy' not in data: return (False, "Error getting proxy dns. 'update_proxy_key_not_found'") using_proxy = data['update_update_proxy'] !='disabled' if using_proxy: (success, data) = get_av_config(system_ip, {'update_update_proxy_dns': ''}) if not success: return (False, "Error getting proxy dns") if 'update_update_proxy_dns' not in data: return (False, "Error getting proxy dns. 'update_update_proxy_dns not found'") dns_lookup = data['update_update_proxy_dns'] (success, data) = ansiblemethods.system.network.resolve_dns_name(system_ip, dns_lookup) if not success: return (False, "Error resolving DNS name") return (True, data)
def apimethod_check_asynchronous_command_return_code(system_id, rc_file): """Check the return code of a previously asynchronous request Args: system_ip (str): The system_id where you want to know if the process is running rc_file(str): The return code file Returns: (boolean, str): A tuple containing the result of the execution Examples: apimethod_ansible_check_asynchronous_command_return_code("11111111-1111-1111-1111-1111222244445555", "/var/log/alienvault/update/system_reconfigure.log.rc") (True,0) """ (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: error_msg = "[apimethod_ansible_check_" + \ "asynchronous_command_return_code] " + \ "Error retrieving the system ip " + \ "for the system id %s -> %s" % (system_ip, str(system_ip)) return success, error_msg return ansible_check_asynchronous_command_return_code(system_ip, rc_file)
def apimethod_check_task_status(system_id, tasks): """ Check the status of a given list of tasks. IE: alienvault-update, alienvault-reconfig Args: system_id (str) : The system_id where you want to check if it's running tasks (dict) : The list of tasks to test. Returns: success (bool) : True if successful, False elsewhere task_status (dict) : A dictionary containing job_id, job_status for each task """ task_status = {} success, system_ip = get_system_ip_from_system_id(system_id) if not success: error_msg = "[apimethod_check_task_status] " + \ "Unable to get system ip " + \ "for system id %s: %s" % (system_id, system_ip) api_log.error(error_msg) return False, {} success, task_status = get_task_status(system_id, system_ip, tasks) if not success: error_msg = "[apimethod_check_task_status] " + \ "Unable to get the task status " + \ "for system %s: %s" % (system_id, str(task_status)) api_log.error(error_msg) return False, {} return success, task_status
def get_plugin_sids_package(system_id, md5): """ Check the :system_id: system if its alienvault-plugin-sids package has md5 sum of :md5:. Download the package from remote system. check if not reconfig / update is running. Install package """ # First, check remote md5 rt = False emsg = '' try: result, info = get_plugin_package_info_from_system_id(system_id) if not result: raise Exception("Can't obtain alienvault-plugin-sid info for system %s : %s" % (system_id, str(info))) if info['md5'] != md5: raise Exception("md5 provided doesn't match with stored md5") # Use ansible to download file to temp directory result, ipremote = get_system_ip_from_system_id(system_id) if not result: raise Exception("Can't obtain remote system ip") result, iplocal = get_system_ip_from_local() if not result: raise Exception("Can't obtain local system ip") result, idlocal = get_system_id_from_local() if not result: raise Exception("Can't obtain local system id") # Create a temp file temp = NamedTemporaryFile(delete=True) tempname = temp.name plugin_package = "alienvault-plugin-sids_" + info['version'] + "_all.deb" remote_path = "/var/cache/apt/archives" result, emsg = fetch_if_changed(ipremote, os.path.join(remote_path, plugin_package), iplocal, tempname) if not result: raise Exception("Can't copy remote from %s file name %s Error: %s" % (ipremote, os.path.join(remote_path, plugin_package), emsg)) shutil.copy(tempname, remote_path) # Atomic rename os.rename(os.path.join(remote_path, os.path.basename(tempname)), os.path.join(remote_path, plugin_package)) # Check if we're not updaing / configuring result, status = check_update_and_reconfig_status(idlocal) if not result: raise Exception("Can't check current status reconfig / update") if status['alienvault-update']['job_status'] == 'running': raise Exception("alienvault-update running") if status['alienvault-reconfig']['job_status'] == 'running': raise Exception("alienvault-reconfig running") if status['ossim-reconfig']['job_status'] == 'running': raise Exception("ossim-reconfig running") # Okey, install package result, status = install_debian_package([iplocal], os.path.join(remote_path, plugin_package)) if not result: raise Exception("Can't install %s" % os.path.join(remote_path, plugin_package)) rt = True emsg = '' except Exception as excep: emsg = str(excep) rt = False return (rt, emsg)
def apimethod_check_task_status(system_id, tasks): """ Check the status of a given list of tasks. IE: alienvault-update, alienvault-reconfig Args: system_id (str) : The system_id where you want to check if it's running tasks (dict) : The list of tasks to test. Returns: success (bool) : True if successful, False elsewhere task_status (dict) : A dictionary containing job_id, job_status for each task """ task_status = {} success, system_ip = get_system_ip_from_system_id(system_id) if not success: error_msg = "[apimethod_check_task_status] " + \ "Unable to get system ip " + \ "for system id %s: %s" % (system_id, system_ip) api_log.error(error_msg) return False, {} success, task_status = get_task_status(system_id, system_ip, tasks) if not success: error_msg = "[apimethod_check_task_status] " + \ "Unable to get the task status " + \ "for system %s: %s" % (system_id, str(task_status)) api_log.error(error_msg) return False, {} return success, task_status
def check_update_and_reconfig_status(system_id): """ Check the status of alienvault-update and alienvault-reconfig tasks Args: system_id (str) : The system_id where you want to check if it's running Returns: success (bool) : True if successful, False elsewhere task_status (dict) : A dictionary containing job_id, job_status for each task """ success, system_ip = get_system_ip_from_system_id(system_id) if not success: error_msg = "[check_update_and_reconfig_status] " + \ "Unable to get system ip " + \ "for system id %s: %s" % (system_id, system_ip) api_log.error(error_msg) return False, "" """" This is the list of task to check. the format is the following: { <Name of the task>: {'task': <name of the celery task>, 'process': <name of the process>, 'param_value': <task condition>, 'param_argnum': <position of the condition>} } In this particular case, we check the alienvault-update and alienvault-reconfig. The condition is that the task has to belong to the given system_ip """ t_list = { "alienvault-update": { 'task': 'alienvault_asynchronous_update', 'process': 'alienvault-update', 'param_value': system_ip, 'param_argnum': 0 }, "alienvault-reconfig": { 'task': 'alienvault_asynchronous_reconfigure', 'process': 'alienvault-reconfig', 'param_value': system_ip, 'param_argnum': 0 }, "ossim-reconfig": { 'task': '', 'process': 'ossim-reconfig', 'param_value': system_ip, 'param_argnum': 0 } } (success, tasks_status) = apimethod_check_task_status(system_id, t_list) if not success: error_msg = "[check_update_and_reconfig_status] " + \ "Unable to get system ip " + \ "for system id %s: %s" % (system_id, system_ip) api_log.error(error_msg) return success, tasks_status
def get_jobs_running(system_id='local'): """ Searches a system for running jobs """ # Get system_ip from system id (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: error_msg = "Error retrieving the system ip " + \ "for the system id %s " % system_id + \ "-> %s" % str(system_ip) api_log.error(str(system_ip)) return False, error_msg success, running_tasks = get_running_tasks(system_ip) conf_backup_task = ".".join(["celerymethods", "tasks", "backup_tasks", "backup_configuration_for_system_id"]) conf_backup_file_task = ".".join(["celerymethods", "tasks", "backup_tasks", "get_backup_file"]) reconfigure_task = ".".join(["celerymethods", "jobs", "reconfig", "alienvault_reconfigure"]) as_reconfigure_task = ".".join(["celerymethods", "jobs", "system", "alienvault_asynchronous_reconfigure"]) update_task = ".".join(["celerymethods", "jobs", "system", "alienvault_asynchronous_update"]) task_types_dict = {conf_backup_task: "configuration_backup", conf_backup_file_task: "get_configuration_backup", reconfigure_task: "reconfigure", as_reconfigure_task: "reconfigure", update_task: "update"} jobs_list = [] for dummy_node, task_list in running_tasks.iteritems(): for task in task_list: if task["name"] in task_types_dict.keys(): cond1 = system_id in literal_eval(task['args']) cond2 = "system_id" in literal_eval(task['kwargs']).keys() cond3 = False if cond2: cond3 = literal_eval(task['kwargs'])["system_id"] == system_id if cond1 or (cond2 and cond3): api_log.error("%s\n" % task['args']) aux_job = {"name": task_types_dict[task["name"]], "time_start": int(task["time_start"]), "job_id": task["id"]} jobs_list.append(aux_job) return success, jobs_list
def get_jobs_running(system_id='local'): """ Searches a system for running jobs """ # Get system_ip from system id (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: error_msg = "Error retrieving the system ip " + \ "for the system id %s " % system_id + \ "-> %s" % str(system_ip) api_log.error(str(system_ip)) return False, error_msg success, running_tasks = get_running_tasks(system_ip) conf_backup_task = ".".join([ "celerymethods", "tasks", "backup_tasks", "backup_configuration_for_system_id" ]) conf_backup_file_task = ".".join( ["celerymethods", "tasks", "backup_tasks", "get_backup_file"]) reconfigure_task = ".".join( ["celerymethods", "jobs", "reconfig", "alienvault_reconfigure"]) as_reconfigure_task = ".".join([ "celerymethods", "jobs", "system", "alienvault_asynchronous_reconfigure" ]) update_task = ".".join( ["celerymethods", "jobs", "system", "alienvault_asynchronous_update"]) task_types_dict = { conf_backup_task: "configuration_backup", conf_backup_file_task: "get_configuration_backup", reconfigure_task: "reconfigure", as_reconfigure_task: "reconfigure", update_task: "update" } jobs_list = [] for dummy_node, task_list in running_tasks.iteritems(): for task in task_list: if task["name"] in task_types_dict.keys(): cond1 = system_id in literal_eval(task['args']) cond2 = "system_id" in literal_eval(task['kwargs']).keys() cond3 = False if cond2: cond3 = literal_eval( task['kwargs'])["system_id"] == system_id if cond1 or (cond2 and cond3): api_log.error("%s\n" % task['args']) aux_job = { "name": task_types_dict[task["name"]], "time_start": int(task["time_start"]), "job_id": task["id"] } jobs_list.append(aux_job) return success, jobs_list
def start(self): """ Starts the monitor activity :return: True on success, False otherwise """ self.remove_monitor_data() monitor_data = {} success, system_id = get_system_id_from_local() if not success: return False # Just return if there is no internet connection. if not self.__check_internet_connection__(): logger.error("Cannot connect to the Telemetry Server") monitor_data['telemetry_server_connectivity'] = False self.save_data(system_id, ComponentTypes.SYSTEM, self.get_json_message(monitor_data)) return True # Find the list of connected systems. (result, sensor_dict) = get_systems('Sensor', convert_to_dict=True, exclusive=True) if not result: logger.error("Cannot retrieve connected sensors") return False (result, database_dict) = get_systems('Database', convert_to_dict=True, exclusive=True) if not result: logger.error("Cannot retrieve connected databases") return False system_dict = dict(sensor_dict, **database_dict) result, local_system_id = get_system_id_from_local() if not result: logger.error("Cannot retrieve the local id") return False result, local_system_ip = get_system_ip_from_system_id(local_system_id) if not result: logger.error("Cannot retrieve the local IP address") return False system_dict = dict({local_system_id: local_system_ip}, **system_dict) args = { 'output_type': 'ansible', 'plugin_list': ','.join(self.__strike_zone_plugins), 'verbose': 2 } ansible_output = get_doctor_data(system_dict.values(), args) if ansible_output.get('dark'): logger.error('Cannot collect telemetry data: %s' % str(ansible_output.get('dark'))) return False return self.__send_data__(local_system_id, ansible_output)
def get(system_id, no_cache=False): """ Get information about a single system """ (success, ip_addr) = ret = get_system_ip_from_system_id(system_id) if not success: return ret return get_system_setup_data(ip_addr)
def get_traffic_stats (system_id): """ Get traffic statistics for a system. """ (success, ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret return ansiblemethods.system.network.get_iface_stats(ip)
def get(system_id, no_cache=False): """ Get information about a single system """ (success, ip_addr) = ret = get_system_ip_from_system_id(system_id) if not success: return ret return get_system_setup_data(ip_addr)
def get_traffic_stats(system_id): """ Get traffic statistics for a system. """ (success, ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret return ansiblemethods.system.network.get_iface_stats(ip)
def ping_system(system_id, no_cache=False): """ Run an ansible ping in the system """ success, system_ip = get_system_ip_from_system_id(system_id) if not success: raise APICannotResolveSystemIP(system_id=system_id, log="[ping system] {0}".format(str(system_ip))) reachable, msg = ansible_ping_system(system_ip) return reachable
def delete_backups(system_id='local', backup_type='configuration', backup_list=[]): """ Delete backups from the system """ success, system_ip = get_system_ip_from_system_id(system_id) if not success: api_log.error(str(system_ip)) error_msg = "Error retrieving the system ip for the system id " error_msg = error_msg + "%s -> %s" % (system_id, str(system_ip)) return False, error_msg success, files = get_files_in_path(system_ip=system_ip, path=BACKUP_PATH) if not success: return False, files # Report warnings for non-existing backup files existing_backup_list = [] for backup_name in backup_list: backup_name = os.path.basename(backup_name) success, backup_path = secure_path_join(BACKUP_PATH, backup_name) if not success: api_log.error("delete_backups: %s '%s'" % (backup_path, backup_name)) elif backup_path not in files.keys(): api_log.error("delete_backups: %s does not exist" % backup_path) else: existing_backup_list.append(backup_path) # Removing existing backups for backup_path in existing_backup_list: try: success, msg = remove_file(host_list=[system_ip], file_name=backup_path) if not success: api_log.error(str(msg)) error_msg = "Error removing %s " % backup_path error_msg = error_msg + "from system %s" % system_ip return False, error_msg except Exception as e: api_log.error("delete_backups Error: %s" % str(e)) error_msg = "Error trying to delete the backup '%s'" % backup_name error_msg = ": %s" % str(e) return False, error_msg try: get_backup_list(system_id=system_id, backup_type=backup_type, no_cache=True) except Exception as e: error_msg = "Error when trying to flush the cache " \ "after deleting backups: %s" % str(e) api_log.error(error_msg) return success, ''
def delete_backups(system_id='local', backup_type='configuration', backup_list=None): """ Delete backups from the system """ if backup_list is None: backup_list = [] success, system_ip = get_system_ip_from_system_id(system_id) if not success: api_log.error(str(system_ip)) error_msg = "Error retrieving the system ip for the system id %s -> %s" % (system_id, str(system_ip)) return False, error_msg success, files = get_files_in_path(system_ip=system_ip, path=BACKUP_PATH) if not success: return False, files # Report warnings for non-existing backup files existing_backup_list = [] backup_name = '' for backup_name in backup_list: backup_name = os.path.basename(backup_name) success, backup_path = secure_path_join(BACKUP_PATH, backup_name) if not success: api_log.error("delete_backups: %s '%s'" % (backup_path, backup_name)) elif backup_path not in files.keys(): api_log.error("delete_backups: %s does not exist" % backup_path) else: existing_backup_list.append(backup_path) # Removing existing backups for backup_path in existing_backup_list: try: success, msg = remove_file(host_list=[system_ip], file_name=backup_path) if not success: api_log.error(str(msg)) error_msg = "Error removing %s from system %s " % (backup_path, system_ip) return False, error_msg except Exception as e: api_log.error("delete_backups Error: %s" % str(e)) error_msg = "Error trying to delete the backup '%s': %s" % (backup_name, str(e)) return False, error_msg try: get_backup_list(system_id=system_id, backup_type=backup_type, no_cache=True) except Exception as e: error_msg = "Error when trying to flush the cache after deleting backups: %s" % str(e) api_log.error(error_msg) return success, ''
def set_system_sensor_configuration(system_id, set_values): (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: return (False, system_ip) (success, config_values) = set_av_config(system_ip, set_values) if not success: api_log.error("system: set_config_general error: " + str(config_values)) return (False, "Cannot set general configuration info: %s" % str(config_values)) return True, "OK"
def put_interface (system_id, iface, promisc): """ Modify network interface properties (currently, only sets promisc mode) """ # Flush the cache "sensor_network" flush_cache(namespace="sensor_network") (success, ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret return ansiblemethods.system.network.set_iface_promisc_status (ip, iface, promisc)
def ping_system(system_id, no_cache=False): """ Run an ansible ping in the system """ success, system_ip = get_system_ip_from_system_id(system_id) if not success: raise APICannotResolveSystemIP(system_id=system_id, log="[ping system] {0}".format( str(system_ip))) reachable, msg = ansible_ping_system(system_ip) return reachable
def status_tunnel(system_id, no_cache=False): """ Get the status of tunnels in system :system_id: """ (success, system_ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret (success, ret) = ans_status_tunnel(system_ip) if not success: api_log.error("system: status_tunnel: " + str(ret)) return False, str(ret) return True, ret
def check_update_and_reconfig_status(system_id): """ Check the status of alienvault-update and alienvault-reconfig tasks Args: system_id (str) : The system_id where you want to check if it's running Returns: success (bool) : True if successful, False elsewhere task_status (dict) : A dictionary containing job_id, job_status for each task """ success, system_ip = get_system_ip_from_system_id(system_id) if not success: error_msg = "[check_update_and_reconfig_status] " + \ "Unable to get system ip " + \ "for system id %s: %s" % (system_id, system_ip) api_log.error(error_msg) return False, "" """" This is the list of task to check. the format is the following: { <Name of the task>: {'task': <name of the celery task>, 'process': <name of the process>, 'param_value': <task condition>, 'param_argnum': <position of the condition>} } In this particular case, we check the alienvault-update and alienvault-reconfig. The condition is that the task has to belong to the given system_ip """ t_list = {"alienvault-update": {'task': 'alienvault_asynchronous_update', 'process': 'alienvault-update', 'param_value': system_ip, 'param_argnum': 0}, "alienvault-reconfig": {'task': 'alienvault_asynchronous_reconfigure', 'process': 'alienvault-reconfig', 'param_value': system_ip, 'param_argnum': 0}, "ossim-reconfig": {'task': '', 'process': 'ossim-reconfig', 'param_value': system_ip, 'param_argnum': 0} } (success, tasks_status) = apimethod_check_task_status(system_id, t_list) if not success: error_msg = "[check_update_and_reconfig_status] " + \ "Unable to get system ip " + \ "for system id %s: %s" % (system_id, system_ip) api_log.error(error_msg) return success, tasks_status
def start(self): """ Starts the monitor activity :return: True on success, False otherwise """ self.remove_monitor_data() monitor_data = {} success, system_id = get_system_id_from_local() if not success: return False # Just return if there is no internet connection. if not self.__check_internet_connection__(): logger.error("Cannot connect to the Telemetry Server") monitor_data['telemetry_server_connectivity'] = False self.save_data(system_id, ComponentTypes.SYSTEM, self.get_json_message(monitor_data)) return True # Find the list of connected systems. (result, sensor_dict) = get_systems('Sensor', convert_to_dict=True, exclusive=True) if not result: logger.error("Cannot retrieve connected sensors") return False (result, database_dict) = get_systems('Database', convert_to_dict=True, exclusive=True) if not result: logger.error("Cannot retrieve connected databases") return False system_dict = dict(sensor_dict, **database_dict) result, local_system_id = get_system_id_from_local() if not result: logger.error("Cannot retrieve the local id") return False result, local_system_ip = get_system_ip_from_system_id(local_system_id) if not result: logger.error("Cannot retrieve the local IP address") return False system_dict = dict({local_system_id: local_system_ip}, **system_dict) args = {'output_type': 'ansible', 'plugin_list': ','.join(self.__strike_zone_plugins), 'verbose': 2} ansible_output = get_doctor_data(system_dict.values(), args) if ansible_output.get('dark'): logger.error('Cannot collect telemetry data: %s' % str(ansible_output.get('dark'))) return False return self.__send_data__(local_system_id, ansible_output)
def network_status(system_id, no_cache=False): """ Return the network facts. Args: system_id (str): A valid uuid or local no_cache (bool): Not used, but we need it declared to make happy the @use_cache decorator Returns: A tuple (success, data) where *success* is a boolean informing the success (True) or failure (False) of the call the *data* member return the network facts as a dict. On error, a message about it is returned in the *data* field. """ success, system_ip = get_system_ip_from_system_id(system_id) if not success: return False, system_ip success, ifaces = get_iface_list(system_ip) if success: # Get the iface disk # ifaces = setup_data['ansible_interfaces'] # ipv4default = setup_data['ansible_default_ipv4'] # Get the network_status_facts success, facts = ans_network_status(system_ip) if success: for iface in facts['interfaces'].keys(): if iface in ifaces: # iface_data = setup_data['ansible_' + iface] if ifaces[iface].get('ipv4', None) is not None: facts['interfaces'][iface]['ipv4'] = ifaces[iface][ 'ipv4'] facts['interfaces'][iface]['role'] = ifaces[iface]['role'] # Add the a "UP" flags # if iface_data['active'] is True: # facts.data['interfaces'][iface]['status'] = 'UP' # else: # facts.data['interfaces'][iface]['status'] = 'DOWN' # Check gateway # if ipv4default.get('interface', None) == iface and ipv4default.get('gateway', None) is not None: # facts.data['gateway'] = ipv4default.get('gateway') pass return True, facts else: return False, facts else: return False, ifaces
def set_system_sensor_configuration(system_id, set_values): (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: return (False, system_ip) (success, config_values) = set_av_config(system_ip, set_values) if not success: api_log.error("system: set_config_general error: " + str(config_values)) return (False, "Cannot set general configuration info: %s" % str(config_values)) return True, "OK"
def get_interfaces(system_id, no_cache=False): """ Return a list of the system network interfaces and its properties. """ (success, ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret (success, ifaces) = ret = ansiblemethods.system.network.get_iface_list(ip) if not success: return ret return (True, ifaces)
def get_interfaces(system_id, no_cache=False): """ Return a list of the system network interfaces and its properties. """ (success, ip) = ret = get_system_ip_from_system_id (system_id) if not success: return ret (success, ifaces) = ret = ansiblemethods.system.network.get_iface_list(ip) if not success: return ret return (True, ifaces)
def backup_configuration_for_system_id(system_id='local', method="auto"): """ Task to run configuration backup for system """ result, system_ip = get_system_ip_from_system_id(system_id) if not result: return False success, msg = make_system_backup(system_id=system_id, backup_type='configuration', rotate=False, retry=False, method=method) return success, msg
def get_last_log_lines(system_id, log_file, lines): """Get a certain number of log lines from a given log file Args: system_id (str): String with system id (uuid) or local. log_file (str): String with the name of the log file. lines (integer): Integer with the number of lines to display. """ # Get system_ip from system id (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: api_log.error(str(system_ip)) error_msg = "Error retrieving the system ip " + \ "for the system id %s -> %s" % (system_ip, str(system_ip)) return False, error_msg # White list check allowed_files = { 'kern': '/var/log/kern.log', 'auth': '/var/log/auth.log', 'daemon': '/var/log/daemon.log', 'messages': '/var/log/messages', 'syslog': '/var/log/syslog', 'agent_stats': '/var/log/alienvault/agent/agent_stats.log', 'agent': '/var/log/alienvault/agent/agent.log', 'server': '/var/log/alienvault/server/server.log', 'reputation': '/var/log/ossim/reputation.log', 'apache_access': '/var/log/apache2/access.log', 'apache_error': '/var/log/apache2/error.log', 'frameworkd': '/var/log/ossim/frameworkd.log', 'last_update': '/var/log/alienvault/update/last_system_update.rc' } if log_file not in allowed_files: return False, "%s is not a valid key for a log file" % log_file if lines not in [50, 100, 1000, 5000]: error_msg = "%s is not a valid number of lines." % str(lines) + \ "The number of lines be in [50, 100, 1000, 5000]" return False, error_msg (success, msg) = ansible_get_log_lines(system_ip, logfile=allowed_files[log_file], lines=lines) if not success: api_log.error(str(msg)) return False, msg return True, msg
def network_status(system_id, no_cache=False): """ Return the network facts. Args: system_id (str): A valid uuid or local no_cache (bool): Not used, but we need it declared to make happy the @use_cache decorator Returns: A tuple (success, data) where *success* is a boolean informing the success (True) or failure (False) of the call the *data* member return the network facts as a dict. On error, a message about it is returned in the *data* field. """ success, system_ip = get_system_ip_from_system_id(system_id) if not success: return False, system_ip success, ifaces = get_iface_list(system_ip) if success: # Get the iface disk # ifaces = setup_data['ansible_interfaces'] # ipv4default = setup_data['ansible_default_ipv4'] # Get the network_status_facts success, facts = ans_network_status(system_ip) if success: for iface in facts["interfaces"].keys(): if iface in ifaces: # iface_data = setup_data['ansible_' + iface] if ifaces[iface].get("ipv4", None) is not None: facts["interfaces"][iface]["ipv4"] = ifaces[iface]["ipv4"] facts["interfaces"][iface]["role"] = ifaces[iface]["role"] # Add the a "UP" flags # if iface_data['active'] is True: # facts.data['interfaces'][iface]['status'] = 'UP' # else: # facts.data['interfaces'][iface]['status'] = 'DOWN' # Check gateway # if ipv4default.get('interface', None) == iface and ipv4default.get('gateway', None) is not None: # facts.data['gateway'] = ipv4default.get('gateway') pass return True, facts else: return False, facts else: return False, ifaces
def get_last_log_lines(system_id, log_file, lines): """Get a certain number of log lines from a given log file Args: system_id (str): String with system id (uuid) or local. log_file (str): String with the name of the log file. lines (integer): Integer with the number of lines to display. """ # Get system_ip from system id (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: api_log.error(str(system_ip)) error_msg = "Error retrieving the system ip " + \ "for the system id %s -> %s" % (system_ip, str(system_ip)) return False, error_msg # White list check allowed_files = { 'kern': '/var/log/kern.log', 'auth': '/var/log/auth.log', 'daemon': '/var/log/daemon.log', 'messages': '/var/log/messages', 'syslog': '/var/log/syslog', 'agent_stats': '/var/log/alienvault/agent/agent_stats.log', 'agent': '/var/log/alienvault/agent/agent.log', 'server': '/var/log/alienvault/server/server.log', 'reputation': '/var/log/ossim/reputation.log', 'apache_access': '/var/log/apache2/access.log', 'apache_error': '/var/log/apache2/error.log', 'frameworkd': '/var/log/ossim/frameworkd.log', 'last_update': '/var/log/alienvault/update/last_system_update.rc' } if log_file not in allowed_files: return False, "%s is not a valid key for a log file" % log_file if lines not in [50, 100, 1000, 5000]: error_msg = "%s is not a valid number of lines." % str(lines) + \ "The number of lines be in [50, 100, 1000, 5000]" return False, error_msg (success, msg) = ansible_get_log_lines(system_ip, logfile=allowed_files[log_file], lines=lines) if not success: api_log.error(str(msg)) return False, msg return True, msg
def set_config_general(system_id): param_names = [ 'general_admin_dns', 'general_admin_gateway', 'general_admin_ip', 'general_admin_netmask', 'general_hostname', 'general_mailserver_relay', 'general_mailserver_relay_passwd', 'general_mailserver_relay_port', 'general_mailserver_relay_user', 'general_ntp_server', 'firewall_active' ] (success, system_ip) = ret = get_system_ip_from_system_id(system_id) if not success: return make_error(system_ip, 500) set_values = {} for key, value in request.args.iteritems(): if key not in param_names: return make_error("Bad param %s" % key, 400) else: set_values[key] = value (success, config_values) = set_av_config(system_ip, set_values) if not success: current_app.logger.error("system: set_config_general error: " + str(config_values)) return make_error( "Cannot set general configuration info %s" % str(config_values), 500) flush_cache(namespace="system") if 'general_hostname' in set_values: success, msg = db_system_update_hostname( system_id, set_values['general_hostname']) if not success: return make_error("Error setting values: %s" % msg, 500) if 'general_admin_ip' in set_values: success, msg = db_system_update_admin_ip( system_id, set_values['general_admin_ip']) if not success: return make_error("Error setting values: %s" % msg, 500) success, msg = ansible_add_ip_to_inventory( set_values['general_admin_ip']) if not success: return make_error("Error setting the admin IP address", 500) job = alienvault_asynchronous_reconfigure.delay(system_id) return make_ok(job_id=job.id)
def add_system(system_id, password): """ Add a system usign a system id. Already in database """ (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: api_log.error(str(system_ip)) return False, "Error retrieving the system ip for the system id %s -> %s" % (system_ip, str(system_ip)) success, msg = add_system_from_ip(system_ip, password, add_to_database=False) if not success: api_log.error(str(msg)) return False, msg return True, msg
def package_list(system_id): """ Add a system usign a system id. Already in database """ (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: api_log.error(str(system_ip)) return False, "Error retrieving the system ip for the system id %s -> %s" % (system_ip, str(system_ip)) success, msg = ans_package_list(system_ip) if not success: api_log.error(str(msg)) return False, msg return True, msg
def package_list(system_id): """ Add a system usign a system id. Already in database """ (success, system_ip) = get_system_ip_from_system_id(system_id) if not success: api_log.error(str(system_ip)) return False, "Error retrieving the system ip for the system id %s -> %s" % (system_ip, str(system_ip)) success, msg = ans_package_list(system_ip) if not success: api_log.error(str(msg)) return False, msg return True, msg
def get_plugin_package_info_from_system_id(system_id): """ Get the alienvault-plugin-sids version from system with id system_id :param: system_id """ (success, system_ip) = get_system_ip_from_system_id(system_id) if success: (success, info) = get_plugin_package_info(system_ip) if not success: result = (False, "Can't get plugins version/md5 information") else: result = (True, info) else: result = (False, "Bad system id: %s" % str(system_id)) return result
def get_plugin_package_info_from_system_id(system_id): """ Get the alienvault-plugin-sids version from system with id system_id :param: system_id """ (success, system_ip) = get_system_ip_from_system_id(system_id) if success: (success, info) = get_plugin_package_info(system_ip) if not success: result = (False, "Can't get plugins version/md5 information") else: result = (True, info) else: result = (False, "Bad system id: %s" % str(system_id)) return result
def connect_tunnel(system_id, case_id): """ Enable the reverse tunnel on the """ (success, system_ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret (success, ret) = ans_connect_tunnel(system_ip, case_id) if not success: api_log.error("system: connect_tunnel: " + str(ret)) return False, str(ret) (succes, result) = ret = status_tunnel(system_id, no_cache=True) if not success: api_log.error("system: status_tunnel: " + str(result)) return ret return True, ''
def get_interface(system_id, iface): """ Return the properties of a single network interface. """ (success, ip) = ret = get_system_ip_from_system_id (system_id) if not success: return ret (success, ifaces) = ret = ansiblemethods.system.network.get_iface_list(ip) if not success: return ret if not iface in ifaces: return (False, "Invalid network interface") return (True, ifaces[iface])
def get_interface(system_id, iface): """ Return the properties of a single network interface. """ (success, ip) = ret = get_system_ip_from_system_id(system_id) if not success: return ret (success, ifaces) = ret = ansiblemethods.system.network.get_iface_list(ip) if not success: return ret if not iface in ifaces: return (False, "Invalid network interface") return (True, ifaces[iface])