def get_config(self, conf, local=False): """ Retrieves local or merged dictionary of dicts local app context. This function creates parity for use with writeConfFile in splunk.clilib. Should use cli.getMergedConf(), but does not support custom conf files. :param conf: Splunk conf file file name :param local: local config only :return: dictionary of dicts """ cli.getMergedConf() conf = "%s.conf" % conf defaultconfpath = os.path.join(self.dir, "default", conf) stanzaDict = cli.readConfFile(defaultconfpath) if os.path.exists(defaultconfpath) else {} localconfpath = os.path.join(self.dir, "local", conf) if not local: if os.path.exists(localconfpath): localconf = cli.readConfFile(localconfpath) for setting, stanza in localconf.items(): if setting in stanzaDict: stanzaDict[setting].update(stanza) else: stanzaDict[setting] = stanza else: stanzaDict = cli.readConfFile(localconfpath) if os.path.exists(localconfpath) else {} return stanzaDict
def get_config(self, conf, local=False): """ Retrieves local or merged dictionary of dicts local app context. This function creates parity for use with writeConfFile in splunk.clilib. Should use cli.getMergedConf(), but does not support custom conf files. :param conf: Splunk conf file file name :param local: local config only :return: dictionary of dicts """ cli.getMergedConf() conf = "%s.conf" % conf defaultconfpath = os.path.join(self.dir, "default", conf) stanzaDict = cli.readConfFile(defaultconfpath) if os.path.exists( defaultconfpath) else {} localconfpath = os.path.join(self.dir, "local", conf) if not local: if os.path.exists(localconfpath): localconf = cli.readConfFile(localconfpath) for setting, stanza in localconf.items(): if setting in stanzaDict: stanzaDict[setting].update(stanza) else: stanzaDict[setting] = stanza else: stanzaDict = cli.readConfFile(localconfpath) if os.path.exists( localconfpath) else {} return stanzaDict
def stanzaStatusInternal(args, fromCLI): """ Returns boolean based on whether or not a given stanza in a given file has the "disabled" parameter set. """ paramsReq = (ARG_CONFIG, ARG_MODNAME, ARG_STANZA) paramsOpt = (ARG_AUTHSTR, ) comm.validateArgs(paramsReq, paramsOpt, args) returnDict = {"enabled": False} stanza = args[ARG_STANZA] authStr = (ARG_AUTHSTR in args) and args[ARG_AUTHSTR] or None currStatus = comm.getMergedConf(args[ARG_CONFIG]) if stanza in currStatus: returnDict[ "enabled"] = True # found the stanza, let's say it's enabled unless we learn otherwise. if KEY_DISABLED in currStatus[stanza]: if "true" == currStatus[stanza][KEY_DISABLED].lower(): returnDict[ "enabled"] = False # if disabled=true, override what we thought above. returnDict["stanzas"] = currStatus return returnDict
def _get_spark_settings(self): app = op.basename(op.dirname(op.dirname(op.abspath(__file__)))) spark_conf = scc.getMergedConf("spark") spark_settings = {} for stanza in ("spark_settings",): spark_settings.update(spark_conf[stanza]) return spark_settings
def __init__(self, appname, conf_filename): # In unit testing scenario we don't want to use btool if get_test_state(): app_path = os.path.join(get_base_path(), appname) self.config = cli.getAppConf(conf_filename, appname, use_btool=False, app_path=app_path) else: self.config = cli.getMergedConf(conf_filename)
def editClient(args, fromCLI): """ Edits the various options of a deployment client. THIS WORKS ON THE LOCAL FILESYSTEM. We usually don't want this, so this will only be used for certain options. """ paramsReq = () paramsOpt = tuple(optionMapClient.keys()) comm.validateArgs(paramsReq, paramsOpt, args) returnDict = {} if 0 == len(args): raise cex.ArgError, "No changes have been specified." if not module.deplClientStatus({}, fromCLI)["enabled"]: raise cex.ServerState, ERR_CLIENT_DIS currConf = comm.getMergedConf(module.DEPL_CLI_CONFIG) if not module.DEPL_CLI_STANZA in currConf: raise cex.ParsingError, ERR_CLI_STANZA # assuming that we only support one of each of these tags - replace every tag found, and add if non-existent. for arg, val in args.items(): paramName = optionMapClient[arg] # validate the few types of args we recognize, ignore anything else. try: if arg in (ARG_MCASTURI, ARG_DEPSERVURI): validate.checkIPPortOrHostPort(arg, val) elif arg == ARG_MCASTIP: validate.checkIP(arg, val) elif arg == ARG_POLLFREQ: validate.checkPosInt(arg, val) except cex.ValidationError: if "0" != val: # we use 0 to disable these things, i guess. raise # remove if 0. if "0" == val and paramName in currConf[module.DEPL_CLI_STANZA]: currConf[module.DEPL_CLI_STANZA].pop(paramName) # or add/set. else: currConf[module.DEPL_CLI_STANZA][paramName] = val # if we're at this point, *something* has changed. returnDict["restartRequired"] = True comm.writeConfFile( bundle_paths.make_path(module.DEPL_CLI_CONFIG + ".conf"), currConf) if fromCLI: logger.info("Configuration updated.") return returnDict
def editClient(args, fromCLI): """ Edits the various options of a deployment client. THIS WORKS ON THE LOCAL FILESYSTEM. We usually don't want this, so this will only be used for certain options. """ paramsReq = () paramsOpt = tuple(optionMapClient.keys()) comm.validateArgs(paramsReq, paramsOpt, args) returnDict = {} if 0 == len(args): raise cex.ArgError, "No changes have been specified." if not module.deplClientStatus({}, fromCLI)["enabled"]: raise cex.ServerState, ERR_CLIENT_DIS currConf = comm.getMergedConf(module.DEPL_CLI_CONFIG) if not module.DEPL_CLI_STANZA in currConf: raise cex.ParsingError, ERR_CLI_STANZA # assuming that we only support one of each of these tags - replace every tag found, and add if non-existent. for arg, val in args.items(): paramName = optionMapClient[arg] # validate the few types of args we recognize, ignore anything else. try: if arg in (ARG_MCASTURI, ARG_DEPSERVURI): validate.checkIPPortOrHostPort(arg, val) elif arg == ARG_MCASTIP: validate.checkIP(arg, val) elif arg == ARG_POLLFREQ: validate.checkPosInt(arg, val) except cex.ValidationError: if "0" != val: # we use 0 to disable these things, i guess. raise # remove if 0. if "0" == val and paramName in currConf[module.DEPL_CLI_STANZA]: currConf[module.DEPL_CLI_STANZA].pop(paramName) # or add/set. else: currConf[module.DEPL_CLI_STANZA][paramName] = val # if we're at this point, *something* has changed. returnDict["restartRequired"] = True comm.writeConfFile(bundle_paths.make_path(module.DEPL_CLI_CONFIG + ".conf"), currConf) if fromCLI: logger.info("Configuration updated.") return returnDict
def read_config(self): cfg = cli.getMergedConf('arkime') self.elastic_nodes = dict() for stanza in cfg.keys(): if re.match("^elastic:[^:]+$", stanza): elastic, site = stanza.split(':', 1) self.elastic_nodes[site] = list() for server in cfg[stanza]: if re.match("^server\d+$", server): self.elastic_nodes[site].append(cfg[stanza][server])
def apply_tag(self, session_key, result): ''' This function will make the actual API call to launch the WAS scan. ''' # print >> sys.stderr, "INFO Launching WAS scan with settings: %s" % settings # get API server, user and password api_user, api_password = qualysModule.splunkpopulator.utils.getCredentials(session_key) qualys_conf = scc.getMergedConf('qualys') api_server = qualys_conf['setupentity']['api_server'] use_proxy = qualys_conf['setupentity']['use_proxy'] proxy = qualys_conf['setupentity']['proxy_server'] if api_user is None or api_user == '' or \ api_password is None or api_password == '' or \ api_server is None or api_server == '': logger.info("API server/username/password not configured. Exiting.") # self.add_event(["API server/username/password not configured. Exiting."]) self.addevent("API server/username/password not configured. Exiting.", self.action_name) exit(3) api_config = qapi.Client.APIConfig() api_config.username = api_user api_config.password = api_password api_config.serverRoot = api_server if use_proxy == '1': api_config.useProxy = True api_config.proxyHost = proxy qapi.setupClient(api_config) api_client = qapi.client webapp_id_in_result = result.get('webapp_id', None) tag_ids = self.tag_ids.split(",") logger.info("Tag ID(s) to apply: %s" % tag_ids) api_params = self.get_api_params(tag_ids) logger.debug("Parameters to apply tag: %s" % api_params) self.addevent("Making %s request with params %s" % (API_ENDPOINT + webapp_id_in_result, api_params), self.action_name) response = api_client.get(API_ENDPOINT + webapp_id_in_result, api_params, qapi.Client.SimpleAPIResponse()) api_response = response.get_response() logger.info("API response is %s" % api_response) response_root = ET.fromstring(api_response) response_code = response_root.find('responseCode').text self.addevent("API response is %s" % response_code, self.action_name) if response_code == "SUCCESS": self.addevent("Successfully applied tag id(s) %s to webapp id %s" % (self.tag_ids, webapp_id_in_result), self.action_name) return True else: self.addevent("Could not apply tag id(s) %s to webapp id %s" % (self.tag_ids, webapp_id_in_result), self.action_name) return False
def getPoll(args, fromCLI): paramsReq = () paramsOpt = () comm.validateArgs(paramsReq, paramsOpt, args) retDict = {"enabled" : False} currConf = comm.getMergedConf(module.DEPL_CLI_CONFIG) try: retDict["uri"] = currConf[module.DEPL_CLI_STANZA][XML_DEPSERVURI] retDict["enabled"] = True except KeyError: pass if fromCLI: if retDict["enabled"]: logger.info("Deployment Server URI is set to \"%s\"." % retDict["uri"]) else: logger.info("Deployment Server URI is not set.") return retDict
def getMulticast(args, fromCLI): paramsReq = () paramsOpt = () comm.validateArgs(paramsReq, paramsOpt, args) retDict = {"enabled": False} currConf = comm.getMergedConf(module.DEPL_CLI_CONFIG) try: retDict["uri"] = currConf[module.DEPL_CLI_STANZA][XML_MCASTURI] retDict["enabled"] = True except KeyError: pass if fromCLI: if retDict["enabled"]: logger.info("Multicast URI is set to \"%s\"." % retDict["uri"]) else: logger.info("Multicast URI is not set.") return retDict
def _get_service_now_account(self): app = snow_consts.app_name snow_conf = scc.getMergedConf("service_now") snow_account = {} for stanza in ("snow_default", "snow_account", "snow_proxy"): snow_account.update(snow_conf[stanza]) mgr = cred.CredentialManager(self.getSessionKey(), scc.getMgmtUri()) accs = (("url", "username", "password"), ("proxy_url", "proxy_username", "proxy_password")) for (url_k, user_k, pass_k) in accs: url = snow_account[url_k] username = snow_account[user_k] password = snow_account[pass_k] if url and username == snow_consts.encrypted \ and password == snow_consts.encrypted: userpass = mgr.get_clear_password(url, snow_consts.dummy, app) if not userpass: msg = "Failed to get clear credentials for %s" % url _LOGGER.error(msg) raise Exception(msg) username, password = userpass.split(snow_consts.userpass_sep) snow_account[user_k] = username snow_account[pass_k] = password if snow_account["proxy_port"]: snow_account["proxy_port"] = int(snow_account["proxy_port"]) if utils.is_false(snow_account["proxy_enabled"]): snow_account["proxy_url"] = "" snow_account["proxy_port"] = "" snow_url = snow_account["url"] if not snow_url: raise Exception("ServiceNow account has not been setup.") if not snow_url.startswith("https://"): snow_url = "https://%s" % snow_url if not snow_url.endswith("/"): snow_url = "%s/" % snow_url snow_account["url"] = snow_url return snow_account
def _get_service_now_account(self): app = snow_consts.app_name snow_conf = scc.getMergedConf("service_now") snow_account = {} for stanza in ("snow_default", "snow_account", "snow_proxy"): snow_account.update(snow_conf[stanza]) mgr = cred.CredentialManager(self.session_key, scc.getMgmtUri()) accs = (("url", "username", "password"), ("proxy_url", "proxy_username", "proxy_password")) for (url_k, user_k, pass_k) in accs: url = snow_account[url_k] username = snow_account[user_k] password = snow_account[pass_k] if url and username == "<encrypted>" and password == "<encrypted>": userpass = mgr.get_clear_password(url, "dummy", app) if not userpass: self.logger.error("Failed to get clear credentials for %s", url) raise Exception("Failed to get clear credentials" " for {}".format(url)) username, password = userpass.split("``") snow_account[user_k] = username snow_account[pass_k] = password if snow_account["proxy_port"]: snow_account["proxy_port"] = int(snow_account["proxy_port"]) if utils.is_false(snow_account["proxy_enabled"]): snow_account["proxy_url"] = "" snow_account["proxy_port"] = "" snow_url = snow_account["url"] if not snow_url: raise Exception("ServiceNow account has not been setup.") if not snow_url.startswith("https://"): snow_url = "https://{}".format(snow_url) if not snow_url.endswith("/"): snow_url = "{}/".format(snow_url) snow_account["url"] = snow_url return snow_account
def get_api_parameters(self): api_params = '' qualysConf = scc.getMergedConf("qualys") if 'extra_was_params' in qualysConf['setupentity'] and qualysConf[ 'setupentity']['extra_was_params'] != '': extra_params_root = ET.fromstring( qualysConf['setupentity']['extra_was_params']) for child in extra_params_root: child_attribs = child.attrib if child_attribs['field'] == 'webApp.id': api_params += "<Criteria field=\"id\" operator=\"%s\">%s</Criteria>" % ( child_attribs['operator'], child.text) # if # for if api_params != '': return "<ServiceRequest><filters>" + api_params + "</filters></ServiceRequest>" else: return '' else: return ''
def stanzaStatusInternal(args, fromCLI): """ Returns boolean based on whether or not a given stanza in a given file has the "disabled" parameter set. """ paramsReq = (ARG_CONFIG, ARG_MODNAME, ARG_STANZA) paramsOpt = (ARG_AUTHSTR,) comm.validateArgs(paramsReq, paramsOpt, args) returnDict = {"enabled" : False} stanza = args[ARG_STANZA] authStr = (ARG_AUTHSTR in args) and args[ARG_AUTHSTR] or None currStatus = comm.getMergedConf(args[ARG_CONFIG]) if stanza in currStatus: returnDict["enabled"] = True # found the stanza, let's say it's enabled unless we learn otherwise. if KEY_DISABLED in currStatus[stanza]: if "true" == currStatus[stanza][KEY_DISABLED].lower(): returnDict["enabled"] = False # if disabled=true, override what we thought above. returnDict["stanzas"] = currStatus return returnDict
if options.api_server: api_server = options.api_server if options.username: api_user = options.username if options.password: api_password = options.password if options.start_date: start_date = options.start_date temp_directory = APP_ROOT + '/tmp' qualysConf = scc.getMergedConf("qualys") appConfig = ApplicationConfiguration() appConfig.load() if proxy is None: proxy = qualysConf['setupentity']['proxy_server'] if api_server is None: #if not passed via CLI argument then load from config file api_server = qualysConf['setupentity']['api_server'] if api_server is None or api_server == '': api_server = raw_input("QG API Server:") if api_user is None or api_user == '':
def handle(self, in_string): textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f}) is_binary_string = lambda bytes: bool(bytes.translate(None, textchars)) debug = "" user = "" result = "" reason = "" form = {"action": "", "path": "", "param1": ""} try: conf = getMergedConf(app_name) in_payload = json.loads(in_string) if in_payload['method'] != "POST": return { 'payload': { "message": "Webservice is working but it must be called via POST" }, 'status': 200 } def runCommand(cmds, this_env, status_codes=[]): p = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, env=this_env) o = p.communicate() status_codes.append(p.returncode) return str(o[0]) + "\n" def runCommandGit(git_output, git_status_codes, env_git, cmds): git_output.append({ "type": "cmd", "content": '$ ' + " ".join(cmds) }) git_output.append({ "type": "out", "content": runCommand(cmds, env_git, git_status_codes) }) git_output.append({ "type": "cmd", "content": 'Ended with code: ' + str(git_status_codes[-1]) }) def runCommandCustom(cmds, env_copy): # TODO timeout after: int(conf["global"]["run_timeout"]) p = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, env=env_copy) o = p.communicate() return str(o[0]) + "\n" def git(message, git_status_codes, git_output, file1, file2=None): if confIsTrue("git_autocommit", False): try: files = [file1] if file2 != None: files.append(file2) cmds = [ 'git', 'diff', '--no-ext-diff', '--quiet', '--exit-code' ] cmds.extend(files) runCommandGit(git_output, git_status_codes, env_git, cmds) if git_status_codes.pop() == 1: git_output[-1]['content'] += ' (There are changes)' cmds = ['git', 'add'] cmds.extend(files) runCommandGit(git_output, git_status_codes, env_git, cmds) cmds = ['git', 'commit', '-uno', '-m', message] runCommandGit(git_output, git_status_codes, env_git, cmds) else: git_output[-1]['content'] += ' (No changes)' except Exception as ex: template = "{0}: {1!r}" git_output.append({ "type": "desc", "content": "Git failed. Is git installed and configured correctly?" }) git_output.append({ "type": "out", "content": template.format(type(ex).__name__, ex.args) }) git_status_codes.append(1) def confIsTrue(param, defaultValue): if param not in conf["global"]: return defaultValue if conf["global"][param].lower().strip() in ("1", "true", "yes", "t", "y"): return True return False git_output = [] git_status_codes = [-1] for formParam in in_payload['form']: form[formParam[0]] = formParam[1] user = in_payload['session']['user'] # dont allow write or run access unless the user makes the effort to change the setting if form['action'] == 'run' and not confIsTrue( "run_commands", False): reason = "missing_perm_run" elif ((form['action'] in [ 'delete', 'rename', 'newfolder', 'newfile', 'fileupload', 'fileuploade' ]) or (form['action'] == "save" and form['path'] != "") ) and not confIsTrue("write_access", False): reason = "missing_perm_write" elif form['action'] == "save" and form[ 'path'] == "" and confIsTrue("hide_settings", False): reason = "config_locked" else: env_copy = os.environ.copy() env_git = env_copy.copy() if confIsTrue("git_autocommit", False): git_output.append({ "type": "out", "content": "cwd = " + os.getcwd() + "\n" }) try: git_autocommit_dir = conf["global"][ "git_autocommit_dir"].strip("\"") if git_autocommit_dir != "": env_git["GIT_DIR"] = os.path.join( SPLUNK_HOME, git_autocommit_dir) git_output.append({ "type": "out", "content": "GIT_DIR=" + os.path.join(SPLUNK_HOME, git_autocommit_dir) }) except KeyError: pass try: git_autocommit_work_tree = conf["global"][ "git_autocommit_work_tree"].strip("\"") if git_autocommit_work_tree != "": env_git["GIT_WORK_TREE"] = os.path.join( SPLUNK_HOME, git_autocommit_work_tree) git_output.append({ "type": "out", "content": "GIT_WORK_TREE=" + os.path.join( SPLUNK_HOME, git_autocommit_work_tree) }) except KeyError: pass # when calling read or write with an empty argument it means we are trying to change the config if (form['action'] == 'read' or form['action'] == 'save') and form['path'] == "": localfolder = os.path.join(os.path.dirname(__file__), '..', 'local') form['path'] = os.path.join(os.path.dirname(__file__), '..', 'local', app_name + '.conf') if not os.path.exists(localfolder): os.makedirs(localfolder) if not os.path.exists(form['path']): shutil.copyfile( os.path.join(os.path.dirname(__file__), '..', 'default', app_name + '.conf.example'), form['path']) if form['action'][:5] == 'btool' or form[ 'action'] == 'run' or form['action'] == 'init' or form[ 'action'][:3] == 'git': system = platform.system() os.chdir(SPLUNK_HOME) if system != "Windows" and system != "Linux" and system != "Darwin": reason = "Unable to run commands on this operating system: " + system else: if system == "Windows": cmd = "bin\\splunk" else: cmd = "./bin/splunk" if form['action'] == 'init': result = {} result['files'] = runCommand( [cmd, 'btool', 'check', '--debug'], env_copy) result['conf'] = conf elif form['action'] == 'btool-check': result = runCommand( [cmd, 'btool', 'check', '--debug'], env_copy) result = result + runCommand( [cmd, 'btool', 'find-dangling'], env_copy) result = result + runCommand( [cmd, 'btool', 'validate-strptime'], env_copy) result = result + runCommand( [cmd, 'btool', 'validate-regex'], env_copy) elif form['action'] == 'btool-list': result = runCommand([ cmd, 'btool', form['path'], 'list', '--debug' ], env_copy) elif form['action'] == 'git-log': os.chdir(form['path']) result = runCommand( ['git', 'log', '--stat', '--max-count=100'], env_git) elif form['action'] == 'git-history': os.chdir(os.path.join(SPLUNK_HOME, form['param1'])) result += runCommand([ 'git', 'log', '--follow', '-p', '--', os.path.join(SPLUNK_HOME, form['path']) ], env_git) elif form['action'] == 'run': # dont need to check if we are inside Splunk dir. User can do anything with run command anyway. file_path = os.path.join(SPLUNK_HOME, form['param1']) os.chdir(file_path) result = runCommandCustom(form['path'], env_copy) else: if form['action'][:4] == 'spec': spec_path = os.path.join(SPLUNK_HOME, 'etc', 'system', 'README', form['path'] + '.conf.spec') if os.path.exists(spec_path): with open(spec_path, 'r') as fh: result = fh.read() apps_path = os.path.join(SPLUNK_HOME, 'etc', 'apps') for d in os.listdir(apps_path): spec_path = os.path.join( apps_path, d, 'README', form['path'] + '.conf.spec') if os.path.exists(spec_path): with open(spec_path, 'r') as fh: result = result + fh.read() else: base_path_abs = str( os.path.abspath(os.path.join(SPLUNK_HOME))) file_path = os.path.join(SPLUNK_HOME, form['path']) file_path_abs = str(os.path.abspath(file_path)) if file_path_abs.find(base_path_abs) != 0: reason = "Unable to access path [" + file_path_abs + "] out of splunk directory [" + base_path_abs + "]" else: if form['action'] == 'save': if os.path.isdir(file_path): reason = "Cannot save file as a folder" elif not os.path.exists(file_path): reason = "Cannot save to a file that does not exist" else: os.chdir(os.path.dirname(file_path)) git_output.append({ "type": "desc", "content": "Committing file before saving changes" }) git("unknown", git_status_codes, git_output, file_path) with open(file_path, "wb") as fh: fh.write(form['file']) git_output.append({ "type": "desc", "content": "Committing file after saving changes" }) git(user + " save ", git_status_codes, git_output, file_path) elif form['action'] == 'fs': def pack(base, path, dirs, files): if len(path) == 0: for i in dirs: base[i] = {} base["."] = files else: pack(base[path[0]], path[1:], dirs, files) result = {} cut = len(SPLUNK_HOME.split(os.path.sep)) depth = int(conf["global"]["cache_file_depth"]) for root, dirs, files in os.walk(SPLUNK_HOME): paths = root.split(os.path.sep)[cut:] pack(result, paths, dirs, files) if len(paths) >= depth: del dirs[:] elif form['action'] == 'read': if os.path.isdir(file_path): result = [] for f in os.listdir(file_path): if os.path.isdir( os.path.join(file_path, f)): # for sorting result.append("D" + f) else: result.append("F" + f) else: fsize = os.path.getsize( file_path) / 1000000 if fsize > int( conf["global"]["max_file_size"]): reason = "File too large to open. File size is " + str( fsize ) + " MB and the configured limit is " + conf[ "global"]["max_file_size"] + " MB" else: with open(file_path, 'r') as fh: result = fh.read() if is_binary_string(result): reason = "unable to open binary file" elif form['action'] == 'delete': os.chdir(os.path.dirname(file_path)) git_output.append({ "type": "desc", "content": "Committing file before it is deleted" }) git("unknown", git_status_codes, git_output, file_path) if os.path.isdir(file_path): shutil.rmtree(file_path) else: os.remove(file_path) git_output.append({ "type": "desc", "content": "Deleting file" }) git(user + " deleted ", git_status_codes, git_output, file_path) elif form['action'] == 'fileupload': os.chdir(file_path) if os.path.exists(form['param1']): reason = "File already exists" #elif re.search(r'[^A-Za-z0-9_\- \.\(\)]', form['param1']): # reason = "Uploaded filename contains invalid characters" else: with open(form['param1'], "wb") as fh: idx = form['file'].index(',') fh.write( base64.b64decode( form['file'][idx:])) git_output.append({ "type": "desc", "content": "Adding uploaded file" }) git(user + " uploaded ", git_status_codes, git_output, form['param1']) else: if re.search(r'[^A-Za-z0-9_\- \.\(\)]', form['param1']): reason = "New name contains invalid characters" elif form['action'] == 'rename': new_path = os.path.join( os.path.dirname(file_path), form['param1']) if os.path.exists(new_path): reason = "That already exists" else: os.chdir(os.path.dirname(file_path)) git_output.append({ "type": "desc", "content": "Committing file before renaming" }) git("unknown", git_status_codes, git_output, file_path) os.rename(file_path, new_path) git_output.append({ "type": "desc", "content": "Committing renamed file" }) git(user + " renamed", git_status_codes, git_output, new_path, file_path) else: new_path = os.path.join( file_path, form['param1']) if os.path.exists(new_path): reason = "That already exists" elif form['action'] == 'newfolder': os.makedirs(new_path) elif form['action'] == 'newfile': open(new_path, 'w').close() os.chdir(os.path.dirname(new_path)) git(user + " new", git_status_codes, git_output, new_path) # result may contain binary if there is an attempted read on a binary file. This will break the json if reason != "": result = "" if not confIsTrue("git_autocommit", False): git_output = "" logger.info( 'user={} action={} item="{}" param1="{}" reason="{}"'.format( user, form['action'], form['path'], form['param1'], reason)) return { 'payload': { 'result': result, 'reason': reason, 'debug': debug, 'git': git_output, 'git_status': max(git_status_codes) }, 'status': 200 } except Exception as ex: template = "An exception of type {0} occurred. Arguments:\n{1!r}" message = template.format(type(ex).__name__, ex.args) logger.info('user={} action={} item="{}" param1="{}"'.format( user, form['action'], form['path'], form['param1'])) logger.warn('caught error {} debug={}'.format(message, debug)) return { 'payload': { 'reason': message, 'debug': debug }, 'status': 200 }
# Import the required libraries import xmlrpclib import sys import os import nCircleAPI from splunk.clilib.cli_common import getMergedConf # nCircle Variables config_file = "ncircle" counter_file = "/opt/splunk/etc/apps/SplunkForNCircle/bin/counter_file.txt" latest_counter = 211000 _TIMEOUT = 5 for key in getMergedConf(config_file).keys(): try: host = getMergedConf(config_file)[key]["host"] user = getMergedConf(config_file)[key]["username"] password = getMergedConf(config_file)[key]["password"] # print host # print user # print password jsonStruct = nCircleAPI._getConfigFile(counter_file) if not jsonStruct.get(host + user + "_maxId"): jsonStruct[host + user + "_maxId"] = latest_counter try: # Connect to the server and login (server, session) = nCircleAPI._login(host, user, password)
# Import the required libraries import xmlrpclib import xmltodict import json import sys import os import nCircleAPI import time from splunk.clilib.cli_common import getMergedConf deviceProfilers = ['DP.9','DP.10','DP.11','DP.13','DP.23','DP.25','DP.26','DP.27','DP.28','DP.29','DP.33','DP.35'] dpList = {} config_file = "ncircle" _TIMEOUT = 5 for key in getMergedConf(config_file).keys(): try: host = getMergedConf(config_file)[key]['host'] user = getMergedConf(config_file)[key]['username'] password = getMergedConf(config_file)[key]['password'] try: # Connect to the server and login (server, session) = nCircleAPI._login(host, user, password) for dp in deviceProfilers: deviceProfilerDetail = nCircleAPI._getItem(server, session, dpList, dp) deviceProfilerDetail['authenticationKey'] = 'anonymized' print "{", nCircleAPI._printJson(deviceProfilerDetail) print "}" nCircleAPI._logout(server, session)
def _run(self, configuration_dict=None): """ :rtype : object :type configuration_dict: dict :param api_user: :param api_password: :param configuration_dict: """ qlogger.info("Start") try: qualysConf = scc.getMergedConf("qualys") output_to_stdout = True detection_logger = logging.getLogger('WAS_DETECTIONS') start_time = datetime.utcnow() preserve_api_output = self.preserve_api_output kbPopulator = QualysKnowledgebasePopulator() kbPopulator.preserve_api_output = preserve_api_output log_detections = bool_value( qualysConf['setupentity']['log_individual_findings']) log_host_summary = bool_value( qualysConf['setupentity']['log_webapp_summary']) if log_detections or log_host_summary: detection_configuration = WASDetectionPopulatorConfiguration( kbPopulator, detection_logger) cp_last_run_datetime = self.checkpointData.get( 'last_run_datetime', self.STARTDATE) if cp_last_run_datetime: try: qlogger.info("WAS findings were last fetched on %s", cp_last_run_datetime) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "WAS findings were last fetched on %s" % cp_last_run_datetime) last_fetched_date_time = datetime.strptime( cp_last_run_datetime, '%Y-%m-%dT%H:%M:%SZ') detection_configuration.add_detection_api_filter( 'lastTestedDate', 'GREATER', cp_last_run_datetime) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Fetching WAS findings data for web apps which were scanned after %s" % cp_last_run_datetime) qlogger.info( "Fetching WAS findings data for Hosts which were scanned after %s", cp_last_run_datetime) except ValueError: qlogger.error("Incorrect date format found: %s", last_fetched_date_time) qlogger.info("Fetching all WAS detection data") # setup custom detection api parameters extra_params = None if 'extra_was_params' in qualysConf[ 'setupentity'] and qualysConf['setupentity'][ 'extra_was_params'] != '': qlogger.info("Parsing extra WAS parameter string:%s", qualysConf['setupentity']['extra_was_params']) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Parsing extra WAS parameter string:%s" % qualysConf['setupentity']['extra_was_params']) try: extra_params_root = ET.fromstring( qualysConf['setupentity']['extra_was_params']) for child in extra_params_root: child_attribs = child.attrib qlogger.info("Adding WAS param: %s %s %s", child_attribs['field'], child_attribs['operator'], child.text) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Adding WAS param: %s %s %s" % (child_attribs['field'], child_attribs['operator'], child.text)) detection_configuration.add_detection_api_filter( child_attribs['field'], child_attribs['operator'], child.text) except ValueError, e: qlogger.info( "Error parsing extra WAS parameters: %s Error: %s", qualysConf['setupentity']['extra_was_params'], e.message) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Error parsing extra WAS parameters: %s Error: %s" % (qualysConf['setupentity']['extra_was_params'], e.message)) detection_configuration.host = self.HOST detection_configuration.index = self.INDEX detection_configuration.preserve_api_output = preserve_api_output detection_configuration.collect_advanced_host_summary = True detection_configuration.log_host_detections = log_detections detection_configuration.log_host_summary = log_host_summary detection_configuration.truncation_limit = 5000 detection_configuration.log_host_details_in_detection = bool_value( qualysConf['setupentity'] ['log_host_details_in_detections']) try: # configure which fields to log for HOSTSUMMARY events if self.settings.get('host_summary_fields'): WASDetectionPopulator.host_fields_to_log = self.settings.get( 'host_summary_fields') # Setup which fields to log for HOSTVULN events if self.settings.get('detection_fields'): WASDetectionPopulator.detection_fields_to_log = self.settings.get( 'detection_fields') use_multi_threading = False if 'use_multi_threading_for_was' in qualysConf[ 'setupentity']: use_multi_threading = bool_value( qualysConf['setupentity'] ['use_multi_threading_for_was']) total_logged = 0 if use_multi_threading: num_threads = int( qualysConf['setupentity']['num_threads_for_was']) if num_threads < 0 or num_threads > 10: num_threads = 2 config = {"num_threads": num_threads} qlogger.info( "Running in multi-thread mode with num_threads=%s", num_threads) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Running in multi-thread mode with num_threads=%s" % num_threads) wfc = WASFindingsFetchCoordinator( num_threads, detection_configuration) wfc.coordinate() total_logged = wfc.getLoggedHostsCount() else: qlogger.info("Running in single thread mode") qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Running in single thread mode") detection_api_populator = WASDetectionPopulator( detection_configuration) detection_api_populator.run() total_logged = detection_api_populator.get_host_logged_count qlogger.info("Done loading detections for %d hosts.", total_logged) # store date/time when data pull was started, only if atlease one host was logged if total_logged > 0: self.checkpointData[ 'last_run_datetime'] = start_time.strftime( '%Y-%m-%dT%H:%M:%SZ') qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "setting checkpointData last_run_datetime to %s" % self.checkpointData['last_run_datetime']) first_run = self.checkpointData.get('first_run', True) if first_run: self.checkpointData['first_run'] = False self.saveCheckpoint() except BasePopulatorException, e: qlogger.error(e.message) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", e.message)
def _run(self, configuration_dict=None): """ :rtype : object :type configuration_dict: dict :param api_user: :param api_password: :param configuration_dict: """ qlogger.info("Start") try: output_to_stdout = True # first read from config file then check if an option was provided on command line log_output_directory = self.settings.get('log_output_directory', None) if log_output_directory is not None and log_output_directory != '': output_to_stdout = False detection_logger = logging.getLogger('HOST_DETECTIONS') start_time = datetime.utcnow() preserve_api_output = self.preserve_api_output kbPopulator = QualysKnowledgebasePopulator() kbPopulator.preserve_api_output = preserve_api_output log_detections = bool_value( self.settings.get('log_detections', True)) log_host_summary = bool_value( self.settings.get('log_host_summary', True)) if log_detections or log_host_summary: if not qapi.client.qweb_version or qapi.client.qweb_version < 8.3: qlogger.info( 'Fetching KB as part of detections because qweb_version=%s is less than 8.3.', qapi.client.qweb_version) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Fetching KB as part of detections becuase qweb_version=%s is less than 8.3" % qapi.client.qweb_version) kbPopulator.run() detection_configuration = HostDetectionPopulatorConfiguration( kbPopulator, detection_logger) detection_configuration.add_detection_api_filter( 'status', 'New,Active,Fixed,Re-Opened') cp_last_run_datetime = self.checkpointData.get( 'last_run_datetime', self.STARTDATE) if cp_last_run_datetime: try: qlogger.info("Last run date time in file is %s", cp_last_run_datetime) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "last run date time in file is %s" % cp_last_run_datetime) last_fetched_date_time = datetime.strptime( cp_last_run_datetime, '%Y-%m-%dT%H:%M:%SZ') detection_configuration.add_detection_api_filter( 'vm_scan_since', cp_last_run_datetime) qlogger.info( "Fetching detection data for Hosts which were scanned after %s", cp_last_run_datetime) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Fetching detection data for Hosts which were scanned after %s" % cp_last_run_datetime) except ValueError: qlogger.error("Incorrect date format found: %s", last_fetched_date_time) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Incorrect date format %s" % last_fetched_date_time) qlogger.info("Fetching all detection data") # setup custom detection api parameters qualysConf = scc.getMergedConf("qualys") extra_params = None if 'detection_params' in qualysConf[ 'setupentity'] and qualysConf['setupentity'][ 'detection_params'] != '': qlogger.info("Parsing extra detection parameter string:%s", qualysConf['setupentity']['detection_params']) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Parsing extra detection string %s" % qualysConf['setupentity']['detection_params']) try: extra_params = json.loads( qualysConf['setupentity']['detection_params']) except ValueError, e: qlogger.info( "Parameters are not in JSON format, parsing as regular URL params: %s. ERROR=%s", qualysConf['setupentity']['detection_params'], e.message) extra_params = urlparse.parse_qs( qualysConf['setupentity']['detection_params']) extra_params = dict( map(lambda (k, v): (k, ','.join(v)), extra_params.iteritems())) if extra_params: for name in extra_params: qlogger.info( "Adding detection param:%s with value%s", name, extra_params[name]) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Adding detection param:%s with value %s" % (name, extra_params[name])) detection_configuration.add_detection_api_filter( name, extra_params[name]) else: qlogger.error( "Error setting extra detection API parameters via string:%s", qualysConf['setupentity']['detection_params']) detection_configuration.host = self.HOST detection_configuration.index = self.INDEX detection_configuration.preserve_api_output = preserve_api_output detection_configuration.collect_advanced_host_summary = True detection_configuration.log_host_detections = log_detections detection_configuration.log_host_summary = log_host_summary detection_configuration.truncation_limit = 5000 detection_configuration.log_host_details_in_detection = bool_value( qualysConf['setupentity'] ['log_host_details_in_detections']) try: # configure which fields to log for HOSTSUMMARY events if self.settings.get('host_summary_fields'): HostDetectionPopulator.host_fields_to_log = self.settings.get( 'host_summary_fields') # Setup which fields to log for HOSTVULN events if self.settings.get('detection_fields'): HostDetectionPopulator.detection_fields_to_log = self.settings.get( 'detection_fields') use_multi_threading = bool_value( qualysConf['setupentity']['use_multi_threading']) total_logged = 0 if use_multi_threading: num_threads = int( qualysConf['setupentity']['num_threads']) if num_threads < 0 or num_threads > 10: num_threads = 2 config = {"num_threads": num_threads} qlogger.info( "Running in multi-thread mode with num_threads=%s", num_threads) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Running in multi-thread mode with num_threads=%s" % num_threads) dfc = DetectonFetchCoordinator( config, detection_configuration) dfc.coordinate() total_logged = dfc.get_host_logged_count else: qlogger.info("Running in single thread mode") qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Running in single thread mode") detection_api_populator = HostDetectionPopulator( detection_configuration) detection_api_populator.run() total_logged = detection_api_populator.get_host_logged_count qlogger.info("Done loading detections for %d hosts.", total_logged) # store date/time when data pull was started, only if atlease one host was logged #TODO: update checkpoint at this point if total_logged > 0: self.checkpointData[ 'last_run_datetime'] = start_time.strftime( '%Y-%m-%dT%H:%M:%SZ') qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "Setting checkpointData last_run_datetime to %s" % self.checkpointData['last_run_datetime']) first_run = self.checkpointData.get('first_run', True) if first_run: self.checkpointData['first_run'] = False self.saveCheckpoint() except BasePopulatorException, e: qlogger.error(e.message) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", e.message)
elif a.startswith("debug"): debug = True error = False # check that everthing is ok... if index == "" and not testmode: results = splunk.Intersplunk.generateErrorResults("Error: you need to supply a valid index name. It is suggested you not use your main index and create a special cache index") if debug: results = splunk.Intersplunk.generateErrorResults("DEBUG: file=%s, path=%s, index=%s, format=%s, marker=%s" % (file, path, index, format, marker) ) # if no error do the work if results == "": results,dummyresults,settings = splunk.Intersplunk.getOrganizedResults() config = cli_common.getMergedConf('indexes') if not index in config: error = True results = splunk.Intersplunk.generateErrorResults("index='" + index + "' does not exist.") if not error and len(results) > 0 : results = writeEventCache(getFilename(file, path, False), results) except Exception, e: results = splunk.Intersplunk.generateErrorResults(str(e)) splunk.Intersplunk.outputResults(results)
def run(): config = getConfig() if ('stanzaDict' not in config or \ 0 == len(config['stanzaDict'])): print "No input(s) configured /enabled. Doing nothing." else: api_user, api_password = qualysModule.splunkpopulator.utils.getCredentials( config['session_key']) qualysConf = scc.getMergedConf("qualys") if qualysConf['setupentity']['enable_debug'] == '1': qualysModule.enableDebug(True) appConfig = ApplicationConfiguration() appConfig.load() api_server = qualysConf['setupentity']['api_server'] useProxy = qualysConf['setupentity']['use_proxy'] proxy = qualysConf['setupentity']['proxy_server'] if (api_user is None or api_user == '' or \ api_password is None or api_password == '' or \ api_server is None or api_server == ''): print "API Server/Username/Password not configured. Exiting." exit(1) apiConfig = qapi.Client.APIConfig() apiConfig.username = api_user apiConfig.password = api_password apiConfig.serverRoot = api_server if useProxy == '1': apiConfig.useProxy = True if proxy != '': apiConfig.proxyHost = proxy else: qlogger.error( 'You have enabled Proxy but Host field is empty. Cannot proceed further.' ) qualysModule.splunkpopulator.utils.printStreamEventXML( "_internal", "You have enabled Proxy but Host field is empty. Cannot proceed further." ) exit(1) # if qapi.setupClient(apiConfig) qapi.client.validate() for stanzaName, stanza in config['stanzaDict'].iteritems(): # print "Running for %s" % stanzaName pureName = stanzaName.replace("qualys://", "") duration = stanza.get("duration", '288') cp = os.path.join(config.get("checkpoint_dir", "./"), pureName) h = stanza.get("host", "localhost") i = stanza.get("index", "main") sd = stanza.get("start_date", "1999-01-01T00:00:00Z") createPIDFile(pureName) if pureName == "knowledge_base": kbPopulator = qualysModule.qualys_log_populator.QualysKBPopulator( settings=appConfig, checkpoint=cp, host=h, index=i, start_date=sd) kbPopulator.populate_lookup_table = True kbPopulator.run() # end of if knowledge_base if pureName == "host_detection": detectionPopulator = qualysModule.qualys_log_populator.QualysDetectionPopulator( settings=appConfig, checkpoint=cp, host=h, index=i, start_date=sd) detectionPopulator.run() # end of if host_ditection if pureName == "was_findings": wasFindingsPopulator = qualysModule.qualys_log_populator.QualysWasDetectionPopulator( settings=appConfig, checkpoint=cp, host=h, index=i, start_date=sd) wasFindingsPopulator.run() # end of if was_findings durationSeconds = qualysModule.splunkpopulator.utils.timeStringToSeconds( duration) time.sleep(int(durationSeconds)) removePIDFile(pureName)