def save(args):
shouldSave = args['saveresults'] == True
if not shouldSave or invalidEventTypeName(args):
return False
etname = args['eventTypeName']
ns = args['namespace']
user = args['username']
etconf = bundle.getConf('eventtypes', None, ns, user)
stanzaname = etname
etconf.createStanza(stanzaname)
# write out each regex to props.conf
etconf[stanzaname]['search'] = args['eventtype'].replace('\n', ' ').replace('\r','')
style = args['eventTypeStyle']
# if the eventtype has a style, set up an custom renderer
if style != "" and style !="None":
priority = args['eventTypePriority']
render_conf = bundle.getConf('event_renderers', None, ns, user)
stanzaname = etname
render_conf.createStanza(stanzaname)
render_conf[stanzaname]['eventtype'] = etname
render_conf[stanzaname]['css_class'] = style
render_conf[stanzaname]['priority'] = priority
# great success!
successmsg = "'%s' is now saved as an eventtype." % etname
addMessage(args, successmsg, CSUCCESS)
args['successmessage'] = successmsg
return True
def save(args):
shouldSave = args['saveresults'] == True
if not shouldSave or invalidEventTypeName(args):
return False
etname = args['eventTypeName']
ns = args['namespace']
user = args['username']
etconf = bundle.getConf('eventtypes', None, ns, user)
stanzaname = etname
etconf.createStanza(stanzaname)
# write out each regex to props.conf
etconf[stanzaname]['search'] = args['eventtype'].replace('\n',
' ').replace(
'\r', '')
style = args['eventTypeStyle']
# if the eventtype has a style, set up an custom renderer
if style != "" and style != "None":
priority = args['eventTypePriority']
render_conf = bundle.getConf('event_renderers', None, ns, user)
stanzaname = etname
render_conf.createStanza(stanzaname)
render_conf[stanzaname]['eventtype'] = etname
render_conf[stanzaname]['css_class'] = style
render_conf[stanzaname]['priority'] = priority
# great success!
successmsg = "'%s' is now saved as an eventtype." % etname
addMessage(args, successmsg, CSUCCESS)
args['successmessage'] = successmsg
return True
def isCloud(sessionKey):
""" Returns true if running on a cloud stack i.e instanceType == 'cloud' """
server_conf = bundle.getConf('server', sessionKey)
if ('instanceType' in server_conf['general']
and server_conf['general']['instanceType'] == INSTANCE_TYPE_CLOUD):
return True
return False
def writeConf(self, confName, stanzaName, settingsDict):
import splunk.bundle as bundle
app = self.appName # always save things to SOME app context.
user = self.context == CONTEXT_APP_AND_USER and self.userName or "-"
overwriteStanzas = not (self.requestedAction == ACTION_EDIT
or self.requestedAction == ACTION_REMOVE)
try:
confObj = bundle.getConf(confName,
sessionKey=self.getSessionKey(),
namespace=app,
owner=user,
overwriteStanzas=overwriteStanzas)
except splunk.ResourceNotFound:
confObj = bundle.createConf(confName,
sessionKey=self.getSessionKey(),
namespace=app,
owner=user)
confObj.beginBatch()
for k, v in settingsDict.items():
if isinstance(v, list):
confObj[stanzaName][k] = str.join(",", v)
else:
confObj[stanzaName][k] = v
confObj.commitBatch()
def update_distsearch(self, host_app, enabled):
temp = bundle.getConf('distsearch', namespace=host_app, owner='nobody')
if enabled:
temp['replicationBlacklist']['nontsyslogmappings'] = os.path.join(
'apps', host_app, 'lookups', 'ntsyslog_mappings.csv')
else:
temp['replicationBlacklist']['nontsyslogmappings'] = ''
def eventTypeExists(args, eventTypeName=None):
if len(args['eventtype']) == 0:
return True
eventtype = args['eventtype']
props = bundle.getConf('eventtypes', None, args['namespace'],
args['username'])
# for each prop stanza
for stanzaname in props.keys():
if eventTypeName == stanzaname:
addMessage(args, "'%s' eventtype already exists." % eventTypeName,
CWARN)
return True
stanza = props[stanzaname]
search = stanza.get("search", "")
if search.strip().lower() == eventtype.strip().lower():
# just be nice and don't warn the poor guy if he didn't modify the default search
if args['rootSearch'] == args['generatedSearch']:
addMessage(
args,
"Click on event Event Type Feature checkboxes below to make a new unique eventtype.",
CMSG)
else:
addMessage(
args,
"The proposed eventtype is already defined as eventtype '%s'."
% stanzaname, CWARN)
return True
return False
def update_distsearch(self, host_app, enabled):
temp = bundle.getConf('distsearch', namespace=host_app, owner='nobody')
if enabled:
temp['replicationBlacklist']['nontsyslogmappings'] = os.path.join('apps', host_app,
'lookups', 'ntsyslog_mappings.csv')
else:
temp['replicationBlacklist']['nontsyslogmappings'] = ''
def setConf(self,
confDict,
filename,
namespace=None,
sessionKey=None,
owner=None):
''' wrapper to bundle.getConf, still necessary for compatibility'''
try:
conf = bundle.getConf(filename,
sessionKey=sessionKey,
namespace=namespace,
owner=owner)
except:
conf = bundle.createConf(filename,
sessionKey=sessionKey,
namespace=namespace,
owner=owner)
for item in confDict.keys():
try:
for k, v in confDict[item].iteritems():
conf[item][k] = v
except AttributeError:
pass
def main():
startTime = time.time()
os.chdir( sys.path[0] ) #'/Applications/splunk.4.1.5/etc/apps/Keynote/bin'
logger = setup_logging("KeynoteRetriever")
logger.info( "action=starting" )
try:
#get the auth
sessionKey = sys.stdin.readline()
namespace = re.findall(r".*[\\/]([^\\/]+)[\\/]bin",sys.path[0])[0]
try:
conf = bundle.getConf('keynote', sessionKey=sessionKey, namespace=namespace, owner='admin') #extract this from sys.path[0], unless there's a better way
user = conf['keynote']['user']
password = base64.decodestring( conf['keynote']['password'].strip() )
except:
logger.error( "Failed to retrieve keynote config. Use the setup screen in the admin interface to configure. " + str(sys.exc_info()[0]) + "\n" )
raise
if user is None or len(user.strip()) == 0 or user == '000000' or password is None or len(password.strip()) == 0:
logger.error( 'Keynote is not yet configured. Use the setup screen in the admin interface.' )
return
retriever = KeynoteRetriever(sessionKey, user, password, os.path.normcase('../local/') + user + '.last', logger)
filename = retriever.determineNextFile()
while filename is not None:
logger.debug( "Next file is %s" , filename )
if filename is not None:
fileStartTime = time.time()
fh = retriever.getXmlHandleFromZip( retriever.getFileFromServer(filename) )
try:
parser = KeynoteParser()
parser.parse_fh( fh , filename )
retriever.updateLast(filename)
logger.info( "action=parsed filename=" + filename + " seconds=" + str( time.time() - fileStartTime ) )
except:
raise
finally:
fh.close()
filename = retriever.determineNextFile(filename)
# filename = None #uncomment this to limit to one file per run
logger.info( "action=finished seconds=" + str( time.time() - startTime ) )
retriever.firstRun()
except:
e = "Unexpected error (stack trace visible in splunkd.log):" + str(sys.exc_info()[0])
logger.error( e )
sys.stderr.write( e + "\n" )
raise
def setup(self):
limits_conf = bundle.getConf('limits', sessionKey=self.getSessionKey())
enableInstallApps = limits_conf['auth']['enable_install_apps']
if ('enable_install_apps' in limits_conf['auth'] and
splunk.util.normalizeBoolean(enableInstallApps)):
self.setReadCapability(EDIT_OR_INSTALL_APPS)
else:
self.setReadCapability(ADMIN_ALL_OBJECTS)
def fieldNameExtractionExists(args, regex, fieldname=None):
sessionkey = args['sessionkey']
namespace = args['namespace']
owner = args['username']
restriction = args['restriction']
messages = args['messages']
if restriction.startswith("sourcetype="):
restriction = restriction[len("sourcetype="):]
fieldError = []
fieldWarn = []
extractionError = []
extractionWarn = []
# print "sessionkey:", sessionkey
# print "namespace:", namespace
# print "owner:", owner
props = bundle.getConf('props', sessionkey, namespace, owner)
ifMyRegexHadNoNames = removeFieldExtractionName(regex)
# for each prop stanza
for stanzaname in props.keys():
stanza = props[stanzaname]
stanzaname = stanzaname.replace("::", '=')
# for each attribute
for attr,val in stanza.items():
# we have an EXTRACTION
if attr.startswith("EXTRACT"):
# if we have a fieldname (we're saving) and we have an extraction already for the fieldname
if fieldname != None and (("?P<%s>" % fieldname) in val or ("?<%s>" % fieldname) in val):
if stanzaname == restriction:
# crap the exact stanza we care about already has this fieldname!!
fieldError.append(stanzaname)
else:
# note the stanza
fieldWarn.append(stanzaname)
else:
ifYouHadNoNames = removeFieldExtractionName(val)
if ifYouHadNoNames == ifMyRegexHadNoNames:
fieldnames = '/'.join(re.findall("\?P?<(.*?)>", val))
if stanzaname == restriction:
extractionError.append((stanzaname, fieldnames))
else:
extractionWarn.append((stanzaname, fieldnames))
if len(fieldError) > 0:
addMessage(messages, _("'%s' is already extracted for %s.") % (fieldname, prettyList(stanzaname)), CERROR)
if len(fieldWarn) > 0:
addMessage(messages, _("Note: '%s' is currently also being extracted for %s") % (fieldname, prettyList(stanzaname)), CMSG)
if len(extractionError) > 0:
pairs = ["%s for %s" % (stanzaname, fieldnames) for fieldnames, stanzaname in extractionError]
addMessage(messages, _("This regex already extracts %s.") % prettyList(pairs), CERROR)
if len(extractionWarn) > 0:
pairs = ["%s for %s" % (stanzaname, fieldnames) for fieldnames, stanzaname in extractionWarn]
addMessage(messages, _("Note: This regex already extracts %s.") % prettyList(pairs), CWARN)
return len(fieldError) > 0 or len(extractionError) > 0
def fieldNameExtractionExists(args, regex, fieldname=None):
sessionkey = args['sessionkey']
namespace = args['namespace']
owner = args['username']
restriction = args['restriction']
messages = args['messages']
if restriction.startswith("sourcetype="):
restriction = restriction[len("sourcetype="):]
fieldError = []
fieldWarn = []
extractionError = []
extractionWarn = []
# print "sessionkey:", sessionkey
# print "namespace:", namespace
# print "owner:", owner
props = bundle.getConf('props', sessionkey, namespace, owner)
ifMyRegexHadNoNames = removeFieldExtractionName(regex)
# for each prop stanza
for stanzaname in props.keys():
stanza = props[stanzaname]
stanzaname = stanzaname.replace("::", '=')
# for each attribute
for attr,val in stanza.items():
# we have an EXTRACTION
if attr.startswith("EXTRACT"):
# if we have a fieldname (we're saving) and we have an extraction already for the fieldname
if fieldname != None and (("?P<%s>" % fieldname) in val or ("?<%s>" % fieldname) in val):
if stanzaname == restriction:
# crap the exact stanza we care about already has this fieldname!!
fieldError.append(stanzaname)
else:
# note the stanza
fieldWarn.append(stanzaname)
else:
ifYouHadNoNames = removeFieldExtractionName(val)
if ifYouHadNoNames == ifMyRegexHadNoNames:
fieldnames = '/'.join(re.findall("\?P?<(.*?)>", val))
if stanzaname == restriction:
extractionError.append((stanzaname, fieldnames))
else:
extractionWarn.append((stanzaname, fieldnames))
if len(fieldError) > 0:
addMessage(messages, _("'%s' is already extracted for %s.") % (fieldname, prettyList(stanzaname)), CERROR)
if len(fieldWarn) > 0:
addMessage(messages, _("Note: '%s' is currently also being extracted for %s") % (fieldname, prettyList(stanzaname)), CMSG)
if len(extractionError) > 0:
pairs = ["%s for %s" % (stanzaname, fieldnames) for fieldnames, stanzaname in extractionError]
addMessage(messages, _("This regex already extracts %s.") % prettyList(pairs), CERROR)
if len(extractionWarn) > 0:
pairs = ["%s for %s" % (stanzaname, fieldnames) for fieldnames, stanzaname in extractionWarn]
addMessage(messages, _("Note: This regex already extracts %s.") % prettyList(pairs), CWARN)
return len(fieldError) > 0 or len(extractionError) > 0
def handleShowConf(confName, sessionKey, namespace, owner):
"""
handles the show config <confName> cmd
"""
conf = getConf(confName,
sessionKey=sessionKey,
namespace=namespace,
owner=owner)
DISPLAY_CHARS['settings'](conf=conf, cmd='show', obj='config')
def setup(self):
limits_conf = bundle.getConf('limits', sessionKey=self.getSessionKey())
enableInstallApps = limits_conf['auth'].get("enable_install_apps",
False)
if (splunk.util.normalizeBoolean(enableInstallApps)):
self.setReadCapability(EDIT_OR_INSTALL_APPS)
return
# default
self.setReadCapability(ADMIN_ALL_OBJECTS)
def getConfStanzas(stanza):
mystanzas = None
try:
filename, app, origstanzaname = parseInternalValues(stanza[INTERNAL_ATTRIBUTE])
if app == "None":
app = None
mystanzas = bundle.getConf(filename, None, app)
except Exception, e:
print "Unable to get stanzas because %s" % e
def getSplunkConf():
results, dummyresults, settings = isp.getOrganizedResults()
namespace = settings.get("namespace", None)
owner = settings.get("owner", None)
sessionKey = settings.get("sessionKey", None)
conf = sb.getConf('jira', namespace=namespace, owner=owner, sessionKey=sessionKey)
stanza = conf.get('jira')
return stanza
def getConf(self, filename, sessionKey=None, namespace=None, owner=None):
''' wrapper to bundle.getConf, still necessary for compatibility'''
try:
return bundle.getConf(filename,
sessionKey=sessionKey,
namespace=namespace,
owner=owner)
except:
return False
def getSplunkConf():
results, dummyresults, settings = isp.getOrganizedResults()
namespace = settings.get("namespace", None)
owner = settings.get("owner", None)
sessionKey = settings.get("sessionKey", None)
conf = sb.getConf('jira', namespace=namespace, owner=owner, sessionKey=sessionKey)
stanza = conf.get('jira')
return stanza
def getConfStanzas(stanza):
mystanzas = None
try:
filename, app, origstanzaname = parseInternalValues(
stanza[INTERNAL_ATTRIBUTE])
if app == "None":
app = None
mystanzas = bundle.getConf(filename, None, app)
except Exception, e:
print "Unable to get stanzas because %s" % e
class RemoteAppsSetup(splunk.rest.BaseRestHandler):
"""
Prepare remote applications management based on configuration settings.
"""
def __init__(self, method, requestInfo, responseInfo, sessionKey):
splunk.rest.BaseRestHandler.__init__(self,
method,
requestInfo,
responseInfo,
sessionKey)
# Default values
self._allowRemote = True
self._login = LOGIN_URL
self._base = DEFAULT_URL
self._agent = None
self._platformInfo = None
try:
platform_info = platform.platform()
os_name = platform.system()
arch = platform.machine()
py_ver = urllib.URLopener().version
with open(os.path.join(bundle_paths.etc(), "splunk.version")) as f:
for i in f:
if i.startswith("VERSION"):
version = i.split("=")[1].strip().strip('"')
elif i.startswith("BUILD"):
build = i.split("=")[1].strip()
self._agent = "Splunkd/%s (%s; version=%s; arch=%s; build=%s; %s)" % (version, os_name, platform_info, arch, build, py_ver)
self._platformInfo = {'version': version, 'platform': os_name}
except Exception, e:
logger.exception(e)
# Manual overrides in server.conf
try:
conf = bundle.getConf("server", self.sessionKey)
s = conf["applicationsManagement"]
if not s.isDisabled():
if s.has_key("allowInternetAccess"):
self._allowRemote = bundle_paths.parse_boolean(s["allowInternetAccess"])
if s.has_key("loginUrl"):
self._login = s["loginUrl"]
if s.has_key("url"):
self._base = s["url"]
if s.has_key("useragent"):
self._agent = s["useragent"]
except:
pass
logger.debug("applicationsManagement.allowInternetAccess = %s" % str(self._allowRemote))
logger.debug("applicationsManagement.loginUrl = %s" % self._login)
logger.debug("applicationsManagement.url = %s" % self._base)
logger.debug("applicationsManagement.useragent = %s" % self._agent)
def handle_GET(self):
if len(self.pathParts) == 3:
confName = self.pathParts[2]
logger.debug("getConf for confName %s" % confName)
conf = bundle.getConf(confName, sessionKey=self.sessionKey)
else:
namespaceName = self.pathParts[2]
confName = self.pathParts[3]
logger.debug("getConf for confName %s and namespaceName %s" %
(confName, namespaceName))
conf = bundle.getConf(confName,
sessionKey=self.sessionKey,
namespace=namespaceName)
output = [XML_MANIFEST]
stanzas = conf.findStanzas('*')
for stanzaname in stanzas:
splits = []
splits = stanzaname.split(':')
logger.debug("splits is: %s" % splits)
if len(splits) == 1:
output.append('\n<%s ' % stanzaname)
else:
id = splits[1]
output.append('\n<%s id="%s" ' % (splits[0], splits[1]))
keys = {}
stanza = conf[stanzaname]
keys = stanza.findKeys('*')
for (value, key) in enumerate(keys):
# skip _raw
if key != "_raw":
output.append('\n\t%s="%s" ' % (key, value))
output.append('/>')
self.response.write(''.join(output))
def readConf(self, confName):
app = self.context != CONTEXT_NONE and self.appName or "-"
user = self.context == CONTEXT_APP_AND_USER and self.userName or "-"
retDict = {}
try:
thing=bundle.getConf(confName, sessionKey=self.getSessionKey(), namespace=app, owner=user)
for s in thing:
retDict[s] = {}
retDict[s].update(thing[s].items())
# it's not "wrong" to request a conf file that doesn't exist, just like with PropertyPages.
except splunk.ResourceNotFound:
pass
return retDict
def getStanzas(self):
# get stanzas
stanzas = bundle.getConf('preview_props',
sessionKey=self._sessionKey,
namespace=self._mynamespace,
owner=self._owner)
# convert to dict obj and not a conf obj tied to modifying conf system
dictstanzas = {}
for name in stanzas.keys():
d = {}
for a, v in stanzas[name].items():
d[a] = v
dictstanzas[name] = d
return dictstanzas
def setup(self):
if self.requestedAction == admin.ACTION_CREATE:
# Let the C++ handler do all the validation work.
self.supportedArgs.addOptArg('*')
if self.customAction == ACTION_PACKAGE:
limits_conf = bundle.getConf('limits',
sessionKey=self.getSessionKey())
enableInstallApps = limits_conf['auth']['enable_install_apps']
if ('enable_install_apps' in limits_conf['auth']
and splunk.util.normalizeBoolean(enableInstallApps)):
self.customActionCap = EDIT_LOCAL_APPS_CAP_EXPR
else:
self.customActionCap = AAO_OR_EDIT_LOCAL_APPS_CAP_EXPR
def handle_GET(self):
if len(self.pathParts) == 3:
confName = self.pathParts[2]
logger.debug("getConf for confName %s" % confName)
conf = bundle.getConf(confName, sessionKey=self.sessionKey)
else:
namespaceName = self.pathParts[2]
confName = self.pathParts[3]
logger.debug("getConf for confName %s and namespaceName %s" % (confName, namespaceName))
conf = bundle.getConf(confName, sessionKey=self.sessionKey, namespace=namespaceName)
output=[XML_MANIFEST]
stanzas = conf.findStanzas('*')
for stanzaname in stanzas:
splits = []
splits = stanzaname.split(':')
logger.debug("splits is: %s" % splits)
if len(splits) == 1:
output.append('\n<%s ' % stanzaname)
else:
id = splits[1]
output.append('\n<%s id="%s" ' % (splits[0], splits[1]))
keys = {}
stanza = conf[stanzaname]
keys = stanza.findKeys('*')
for (value, key) in enumerate(keys):
# skip _raw
if key != "_raw":
output.append('\n\t%s="%s" ' % (key, value))
output.append('/>' )
self.response.write(''.join(output))
def setConf(self, confDict, filename, namespace=None, sessionKey=None, owner=None ):
''' wrapper to bundle.getConf, still necessary for compatibility'''
try:
conf = bundle.getConf(filename, sessionKey=sessionKey,
namespace=namespace, owner=owner)
except:
conf = bundle.createConf(filename, sessionKey=sessionKey,
namespace=namespace, owner=owner)
for item in confDict.keys():
try:
for k, v in confDict[item].iteritems():
conf[item][k] = v
except AttributeError:
pass
def __init__(self, sessionKey, owner, namespace, args):
self.crawlers = []
self.config = {}
if sessionKey == None:
logger.warn("Username/password authorization not given to 'crawl'. Attempting to carry on.")
try:
self.config = bundle.getConf('crawl', sessionKey)
except:
logger.error("Unable to contact the splunk server.")
exit(-1)
self.args = args
self.sessionKey = sessionKey
self.owner = owner
self.namespace = namespace
def readConf(self, confName):
app = self.context != CONTEXT_NONE and self.appName or "-"
user = self.context == CONTEXT_APP_AND_USER and self.userName or "-"
retDict = {}
try:
thing = bundle.getConf(confName,
sessionKey=self.getSessionKey(),
namespace=app,
owner=user)
for s in thing:
retDict[s] = {}
retDict[s].update(thing[s].items())
# it's not "wrong" to request a conf file that doesn't exist, just like with PropertyPages.
except splunk.ResourceNotFound:
pass
return retDict
def write(self, settings):
if not settings or len(settings) == 0:
return
try:
confObj = bundle.getConf(self.confName,
sessionKey=self.sessionKey,
namespace=AppName,
owner='-')
except:
try:
confObj = bundle.createConf(self.confName,
sessionKey=self.sessionKey,
namespace=AppName,
owner='-')
except Exception, e:
logger.error('Error creating new conf file')
return
def writeConf(self, confName, stanzaName, settingsDict):
app = self.appName # always save things to SOME app context.
user = self.context == CONTEXT_APP_AND_USER and self.userName or "-"
overwriteStanzas = not (self.requestedAction == ACTION_EDIT or self.requestedAction == ACTION_REMOVE)
try:
confObj = bundle.getConf( confName, sessionKey=self.getSessionKey(), namespace=app, owner=user,
overwriteStanzas=overwriteStanzas)
except splunk.ResourceNotFound:
confObj = bundle.createConf(confName, sessionKey=self.getSessionKey(), namespace=app, owner=user)
confObj.beginBatch()
for k, v in settingsDict.items():
if isinstance(v, list):
confObj[stanzaName][k] = str.join(",", v)
else:
confObj[stanzaName][k] = v
confObj.commitBatch()
def load(self):
'''
Read config file
'''
config = {}
conf = None
try:
conf = bundle.getConf(self.confName,
sessionKey=self.sessionKey,
namespace=AppName,
owner='-')
except:
return None
for stanza in conf:
if stanza == 'default':
continue
config[stanza] = dict(conf[stanza])
return config
def handle_POST(self):
sessionKey = self.sessionKey
try:
conf = bundle.getConf('app',
sessionKey,
namespace="Splunk_TA_nix",
owner='nobody')
stanza = conf.stanzas['install'].findKeys('is_configured')
if stanza:
if stanza["is_configured"] == "0" or stanza[
"is_configured"] == "false":
conf["install"]["is_configured"] = 'true'
splunk.rest.simpleRequest(
"/apps/local/Splunk_TA_nix/_reload",
sessionKey=sessionKey)
else:
conf["install"]["is_configured"] = 'true'
splunk.rest.simpleRequest("/apps/local/Splunk_TA_nix/_reload",
sessionKey=sessionKey)
except Exception as e:
self.response.write(e)
def getMatchingStanzas(filenames, apps, kvs, tags, options):
stanzas = {}
defaultstanza = None
owner = splunk.auth.getCurrentUser()['name']
print "Owner: %s" % owner
# for each config file type
for filename in filenames:
# for each app
for app in apps:
if app == "*" or app == '-':
app = None
# get stanzas
try:
mystanzas = bundle.getConf(filename, None, app) #, owner)
except:
exit(
"Unable to get configuration for '%s' for the '%s' application. Exiting..."
% (filename, app))
defaultstanza = mystanzas['default']
# for each stanza
for mystanzaName in mystanzas.keys():
if mystanzaName == 'default':
continue
thisstanza = mystanzas[mystanzaName]
# if it matches the kvs and tag requirements
if match(mystanzaName, thisstanza, kvs, tags, options):
# use a dict rather than actual stanza object, which writes to disk at every change!
mystanza = {}
for k, v in thisstanza.items():
# only specify values that are not on default stanza
if k not in defaultstanza or v != defaultstanza[k]:
mystanza[k] = v
# add on internal bookkeeping attributes so we know where to write the changes back out to
# store stanzaname incase the user changes the stanza name
mystanza[INTERNAL_ATTRIBUTE] = '"%s" "%s" "%s"' % (
filename, app, mystanzaName)
stanzas[mystanzaName] = mystanza
return stanzas, defaultstanza
def eventTypeExists(args, eventTypeName=None):
if len(args['eventtype']) == 0:
return True
eventtype = args['eventtype']
props = bundle.getConf('eventtypes', None, args['namespace'], args['username'])
# for each prop stanza
for stanzaname in props.keys():
if eventTypeName == stanzaname:
addMessage(args, "'%s' eventtype already exists." % eventTypeName, CWARN)
return True
stanza = props[stanzaname]
search = stanza.get("search", "")
if search.strip().lower() == eventtype.strip().lower():
# just be nice and don't warn the poor guy if he didn't modify the default search
if args['rootSearch'] == args['generatedSearch']:
addMessage(args, "Click on event Event Type Feature checkboxes below to make a new unique eventtype.", CMSG)
else:
addMessage(args, "The proposed eventtype is already defined as eventtype '%s'." % stanzaname, CWARN)
return True
return False
def getTransactionInfo(tname, **kwargs):
config = bundle.getConf('transactiontypes', **kwargs)
if tname not in config:
raise Exception("Unknown transactiontype: %s" % (tname))
stanza = config[tname]
if 'fields' not in stanza:
raise Exception("Transactiontype has no fields: %s" % (tname))
field_str = stanza['fields']
fields = re.split("[ ,]+", field_str)
base_search = '*'
if 'search' in stanza:
base_search = stanza['search']
maxspan = None
if "maxspan" in stanza:
maxspan = convertSpanToSecs(stanza["maxspan"])
log("FIELDS: %s" % fields)
return base_search, fields, maxspan
def getTransactionInfo(tname, **kwargs):
config = bundle.getConf('transactiontypes', **kwargs)
if tname not in config:
raise Exception("Unknown transactiontype: %s" % (tname))
stanza = config[tname]
if 'fields' not in stanza:
raise Exception("Transactiontype has no fields: %s" % (tname))
field_str = stanza['fields']
fields = re.split("[ ,]+", field_str)
base_search = '*'
if 'search' in stanza:
base_search = stanza['search']
maxspan = None
if "maxspan" in stanza:
maxspan = convertSpanToSecs(stanza["maxspan"])
log("FIELDS: %s" % fields)
return base_search, fields, maxspan
def getMatchingStanzas(filenames, apps, kvs, tags, options):
stanzas = {}
defaultstanza = None
owner = splunk.auth.getCurrentUser()['name']
print "Owner: %s" % owner
# for each config file type
for filename in filenames:
# for each app
for app in apps:
if app == "*" or app == '-':
app = None
# get stanzas
try:
mystanzas = bundle.getConf(filename, None, app) #, owner)
except:
exit("Unable to get configuration for '%s' for the '%s' application. Exiting..." % (filename, app))
defaultstanza = mystanzas['default']
# for each stanza
for mystanzaName in mystanzas.keys():
if mystanzaName == 'default':
continue
thisstanza = mystanzas[mystanzaName]
# if it matches the kvs and tag requirements
if match(mystanzaName, thisstanza, kvs, tags, options):
# use a dict rather than actual stanza object, which writes to disk at every change!
mystanza = {}
for k, v in thisstanza.items():
# only specify values that are not on default stanza
if k not in defaultstanza or v != defaultstanza[k]:
mystanza[k] = v
# add on internal bookkeeping attributes so we know where to write the changes back out to
# store stanzaname incase the user changes the stanza name
mystanza[INTERNAL_ATTRIBUTE] = '"%s" "%s" "%s"' % (filename, app, mystanzaName)
stanzas[mystanzaName] = mystanza
return stanzas, defaultstanza
def app_configured(self):
sessionKey = self.getSessionKey()
try:
conf = bundle.getConf('app',
sessionKey,
namespace=mmdb_utils.APP_NAME,
owner='nobody')
stanza = conf.stanzas['install'].findKeys('is_configured')
if stanza:
if stanza["is_configured"] == "0" or stanza[
"is_configured"] == "false":
conf["install"]["is_configured"] = 'true'
rest.simpleRequest("/apps/local/{}/_reload".format(
mmdb_utils.APP_NAME),
sessionKey=sessionKey)
else:
conf["install"]["is_configured"] = 'true'
rest.simpleRequest("/apps/local/{}/_reload".format(
mmdb_utils.APP_NAME),
sessionKey=sessionKey)
except Exception as e:
raise Exception(
'Unable to set is_configured parameter in local app.conf file. {}'
.format(e))
def handleShowConf(confName, sessionKey, namespace, owner): """ handles the show config <confName> cmd """ conf = getConf(confName, sessionKey=sessionKey, namespace=namespace, owner=owner) DISPLAY_CHARS['settings'](conf=conf, cmd='show', obj='config')
def saveRule(regex, args):
sessionkey = args['sessionkey']
namespace = args['namespace']
owner = args['username']
fieldname = args['fieldname']
restriction = args['restriction']
messages = args['messages']
shouldSave = args['saveresults'] == True
if not shouldSave or len(regex) == 0:
return False
# add support for multiple field names
fieldnames = fieldname.split(",")
# only one field
if len(fieldnames) == 1:
regex = regex.replace("?P<FIELDNAME>", "?P<%s>" % fieldname)
else:
# split name into multple names (e.g. "status,code,url" then rename FIELDNAME1->status, FIELDNAME2->code, ...
for i, fname in enumerate(fieldnames):
fname = fname.strip()
if invalidFieldName(fname, messages):
return False
regex = regex.replace("?P<FIELDNAME%s>" % (i+1), "?P<%s>" % fname)
# still some unnamed fields!
unnamedCount = regex.count("?P<FIELDNAME")
if unnamedCount > 0:
addMessage(messages, _("Each field must have a name. %s field(s) do not have names." % unnamedCount), CERROR)
return False
# verify user rule has extractions
if args['edited']:
fields = re.findall("\?P<(.*?)>", regex)
if len(fields) == 0:
addMessage(messages, _("Regex '%s' does not contain a named extraction (e.g. '(?P<fieldname>\w+)')"), CERROR)
return False
# set fieldname to pretty name of all extractions it gets
attrSuffix = '-'.join(fields)
else:
attrSuffix = fieldname
# props.conf weirdness -- [sourcetype::name] doesn't
# match. need to use [name]. only for 'sourcetype'.
# other attributes: source, host, and eventtype work
# with their type::name.
# stanza = restriction.replace('=', '::')
stanza = re.sub("^(source|sourcetype|host|eventtype)(=)", "\\1::", restriction)
if stanza.startswith("sourcetype::"):
stanza = stanza[len("sourcetype::"):]
# dequote stanzas. e.g. 'host="localhost"' --> 'host=localhost'
colon = stanza.find('::')
if colon > 0:
stanza = stanza[:colon] + "::" + dequoteVal(stanza[colon+2:])
else:
stanza = dequoteVal(stanza)
if regex.endswith("\\\\"):
regex = regex[:-2] + "[\\\\]"
if fieldNameExtractionExists(args, regex, fieldname):
return False
props = bundle.getConf('props', sessionkey, namespace, owner)
props.createStanza(stanza)
# write out each regex to props.conf
logger.debug("STANZA: [%s] '%s' = '%s'" % (stanza, "EXTRACT-" + attrSuffix, regex))
props[stanza]["EXTRACT-" + attrSuffix] = regex
successmsg = _("'%s' is now extracted as a field.") % fieldname
addMessage(messages, successmsg, CMSG)
args['successmessage'] = successmsg
return True
mon = MonitorInput.all()
mon = mon.filter_by_app(app)
for m in mon:
disabled = normBool(params.get(m.name + '.disabled'))
if disabled:
m.disable()
else:
m.enable()
m.share_global()
self.update_distsearch(host_app, normBool(params.get('optimize_dist_search')))
logger.debug('Splunk Version = %s' % self._get_version())
if self._get_version() <= LooseVersion('4.2.2'):
temp_app = bundle.getConf('app', namespace=host_app, owner='nobody')
temp_app['install']['is_configured'] = 'true'
else:
this_app = App.get(App.build_id(host_app, host_app, user))
this_app.is_configured = True
this_app.passive_save()
logger.info('%s - App setup successful' % host_app)
raise cherrypy.HTTPRedirect(self._redirect(host_app, app, 'success'))
def get_distsearch(self, host_app):
return bundle.getConf('distsearch',
namespace=host_app,
owner='nobody')['replicationBlacklist']['nontsyslogmappings']
def get_distsearch(self, host_app):
return bundle.getConf('distsearch',
namespace=host_app,
owner='nobody')['replicationBlacklist']['nontsyslogmappings']
def __init__(self, method, requestInfo, responseInfo, sessionKey):
splunk.rest.BaseRestHandler.__init__(self, method, requestInfo,
responseInfo, sessionKey)
# Default values
self._allowRemote = True
self._login = LOGIN_URL
self._base = DEFAULT_URL
self._agent = None
self._platformInfo = None
self._supportInProductInstall = True
self._sslpol = bundle_paths.SSLPolicy()
try:
platform_info = platform.platform()
os_name = platform.system()
arch = platform.machine()
py_ver = URLopener().version
with open(os.path.join(bundle_paths.etc(), "splunk.version")) as f:
for i in f:
if i.startswith("VERSION"):
version = i.split("=")[1].strip().strip('"')
elif i.startswith("BUILD"):
build = i.split("=")[1].strip()
self._agent = "Splunkd/%s (%s; version=%s; arch=%s; build=%s; %s)" % (
version, os_name, platform_info, arch, build, py_ver)
self._platformInfo = {'version': version, 'platform': os_name}
except Exception as e:
logger.exception(e)
# Manual overrides in server.conf
try:
conf = bundle.getConf("server", self.sessionKey)
s = conf["applicationsManagement"]
if not s.isDisabled():
if "allowInternetAccess" in s:
self._allowRemote = bundle_paths.parse_boolean(
s["allowInternetAccess"])
if "loginUrl" in s:
self._login = s["loginUrl"]
if "url" in s:
self._base = s["url"]
if "useragent" in s:
self._agent = s["useragent"]
if "caCertFile" in s:
self._sslpol._cafile = bundle_paths.expandvars(
s["caCertFile"])
if "sslCommonNameList" in s:
self._sslpol._sslCommonNameList = bundle_paths.expandvars(
s["sslCommonNameList"])
if "cipherSuite" in s:
self._sslpol._cipherSuite = bundle_paths.expandvars(
s["cipherSuite"])
s = conf["shclustering"]
if not s.isDisabled():
self._supportInProductInstall = False
except Exception as e:
logger.exception(e)
logger.debug("applicationsManagement.allowInternetAccess = %s" %
str(self._allowRemote))
logger.debug("applicationsManagement.loginUrl = %s" % self._login)
logger.debug("applicationsManagement.url = %s" % self._base)
logger.debug("applicationsManagement.useragent = %s" % self._agent)
logger.debug("applicationsManagement.supportInProductInstall = %s" %
str(self._supportInProductInstall))
if self._sslpol._cafile is None:
logger.debug("applicationsManagement.caCertFile = %s" %
str(self._sslpol._cafile))
if self._sslpol._sslCommonNameList is None:
logger.debug("applicationsManagement.sslCommonNameList = %s" %
str(self._sslpol._sslCommonNameList))
if self._sslpol._cipherSuite is None:
logger.debug("applicationsManagement.cipherSuite = %s" %
str(self._sslpol._cipherSuite))
def main():
startTime = time.time()
os.chdir(sys.path[0]) #'/Applications/splunk.4.1.5/etc/apps/Keynote/bin'
logger = setup_logging("KeynoteRetriever")
logger.info("action=starting")
try:
# get the auth
sessionKey = sys.stdin.readline()
namespace = re.findall(r".*[\\/]([^\\/]+)[\\/]bin", sys.path[0])[0]
try:
conf = bundle.getConf(
"keynote", sessionKey=sessionKey, namespace=namespace, owner="admin"
) # extract this from sys.path[0], unless there's a better way
user = conf["keynote"]["user"]
password = base64.decodestring(conf["keynote"]["password"].strip())
except:
logger.error(
"Failed to retrieve keynote config. Use the setup screen in the admin interface to configure. "
+ str(sys.exc_info()[0])
+ "\n"
)
raise
if user is None or len(user.strip()) == 0 or user == "000000" or password is None or len(password.strip()) == 0:
logger.error("Keynote is not yet configured. Use the setup screen in the admin interface.")
return
retriever = KeynoteRetriever(sessionKey, user, password, os.path.normcase("../local/") + user + ".last", logger)
filename = retriever.determineNextFile()
while filename is not None:
logger.debug("Next file is %s", filename)
if filename is not None:
fileStartTime = time.time()
fh = retriever.getXmlHandleFromZip(retriever.getFileFromServer(filename))
try:
parser = KeynoteParser()
parser.parse_fh(fh, filename)
retriever.updateLast(filename)
logger.info("action=parsed filename=" + filename + " seconds=" + str(time.time() - fileStartTime))
except:
raise
finally:
fh.close()
filename = retriever.determineNextFile(filename)
# filename = None #uncomment this to limit to one file per run
logger.info("action=finished seconds=" + str(time.time() - startTime))
retriever.firstRun()
except:
e = "Unexpected error (stack trace visible in splunkd.log):" + str(sys.exc_info()[0])
logger.error(e)
sys.stderr.write(e + "\n")
raise
for m in mon:
disabled = normBool(params.get(m.name + '.disabled'))
if disabled:
m.disable()
else:
m.enable()
m.share_global()
self.update_distsearch(host_app,
normBool(params.get('optimize_dist_search')))
logger.debug('Splunk Version = %s' % self._get_version())
if self._get_version() <= LooseVersion('4.2.2'):
temp_app = bundle.getConf('app',
namespace=host_app,
owner='nobody')
temp_app['install']['is_configured'] = 'true'
else:
this_app = App.get(App.build_id(host_app, host_app, user))
this_app.is_configured = True
this_app.passive_save()
logger.info('%s - App setup successful' % host_app)
raise cherrypy.HTTPRedirect(self._redirect(host_app, app, 'success'))
def get_distsearch(self, host_app):
return bundle.getConf(
'distsearch', namespace=host_app,
owner='nobody')['replicationBlacklist']['nontsyslogmappings']
def get_distsearch(self, host_app):
return bundle.getConf(
'distsearch', namespace=host_app,
owner='nobody')['replicationBlacklist']['nontsyslogmappings']
def getEnv(cls, uri, principal=None):
if cls.namespace == None:
raise HcException(HCERR0000, {
'error':
'HadoopEnvManager.init() must be called before getEnv'
})
from urllib.parse import urlparse
#1. parse uri to get just host:port
host = None
port = None
p = urlparse(uri)
if p.scheme == 'hdfs':
host = p.hostname
port = str(p.port) if p.port != None else None
else:
return dict(os.environ)
if principal == None:
principal = cls.krb5_principal
key = (host, port, principal) #host, port, principal
#2. check if env is in cache
if key in cls.env:
#TODO: for long running processes we'll need to refresh the TGT if it's too old
return dict(cls.env[key]) # copy
#3.use REST to get cluster info from splunkd
rc = None #result cluster
hostport = '%s:%s' % (host, port) if port != None else host
try:
from splunk.bundle import getConf
clusters = getConf('clusters',
sessionKey=cls.sessionKey,
namespace=cls.namespace,
owner=cls.user)
for c in clusters:
if not c == hostport:
continue
rc = clusters[c]
break
except Exception as e:
logger.exception('Failed to get conf info from clusters.conf')
result = None
if rc == None:
raise HcException(HCERR0007, {'cluster': hostport})
else:
hadoop_home = rc['hadoop_home']
java_home = rc['java_home']
app_local = os.path.join(os.environ['SPLUNK_HOME'], 'etc', 'apps',
APP_NAME, 'local')
hadoop_conf_dir = os.path.join(
app_local, 'clusters', makeFileSystemSafe(hostport)
) # use _ instead of : in host:port since : is used as a path separator
if principal == None:
principal = rc.get('kerberos_principal', None)
if principal != None and len(principal.strip()) == 0:
principal = None
#TODO: ensure current user has permission to use the given principal
result = setupHadoopEnv(hadoop_home, java_home, hadoop_conf_dir,
principal)
logger.debug("uri=%s, hadoop_home=%s, java_home=%s" %
(uri, result.get('HADOOP_HOME',
''), result.get('JAVA_HOME', '')))
cls.env[key] = result
return result
