def transform(self, events):
# Bind to Splunk session and initialize variables
service = client.Service(token=self.metadata.searchinfo.session_key)
salt = ""
# Get salt if "salt" was set
if self.salt:
# Check if configuration exists for specified salt
try:
service.confs['inputs']['crypto_settings://{0}'.format(
self.salt)]
except:
raise ValueError(
'Specified salt file "{0}" does not exist. Please check the spelling of your specified salt name or your configured salts.'
.format(self.salt))
# Continue if user is authorized for salt usage
if self.validate_user(service):
salt = self.load_salt(service)
# HASH
#
for event in events:
for fieldname in self.fieldnames:
if fieldname == '_time':
continue
try:
if self.salt:
message = salt.encode(
'utf-8') + event[fieldname].encode('utf-8')
else:
message = event[fieldname].encode('utf-8')
if self.algorithm in [
'md5', 'sha1', 'sha224', 'sha256', 'sha384',
'sha512'
]:
hashmethod = getattr(hashlib, self.algorithm)
event[self.algorithm] = hashmethod(message).hexdigest()
elif sys.version_info >= (3, 0) and self.algorithm in [
'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
'blake2b', 'blake2s'
]:
hashmethod = getattr(hashlib, self.algorithm)
event[self.algorithm] = hashmethod(message).hexdigest()
elif sys.version_info < (3, 0) and self.algorithm in [
'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
'blake2b', 'blake2s'
]:
raise RuntimeWarning(
'Hash algorithm "{0}" is only available when using Python 3 as Splunk\'s Python interpreter.'
.format(self.algorithm))
else:
raise ValueError(
'Invalid hash algorithm "{0}" has been specified.'.
format(self.algorithm))
except Exception as e:
raise RuntimeWarning(
'Failed to hash fields: {0}'.format(e))
yield event
def test_kvstore():
session_key = get_session_key(
context.username,
context.password,
scheme=context.scheme,
host=context.host,
port=context.port,
)
kvstore = client.Service(
scheme=context.scheme,
host=context.host,
port=context.port,
token=session_key,
app=context.app,
owner=context.owner,
autologin=True,
).kvstore
fields = {"id": "string", "name": "string", "user": "******"}
last_ex = None
for i in range(3):
try:
kvstore.create("sessions", fields=fields)
break
except binding.HTTPError as e:
last_ex = e
time.sleep(2**(i + 1))
else:
if last_ex:
raise last_ex
collections = kvstore.list()
collection_data = None
for collection in collections:
if collection.name == "sessions":
collection_data = collection.data
break
assert collection_data
record = {"id": uuid.uuid4().hex, "name": "session1", "user": "******"}
_key = collection_data.insert(json.dumps(record))["_key"]
resp_record = collection_data.query_by_id(_key)
resp_record = {
key: resp_record[key]
for key in resp_record if not key.startswith("_")
}
assert sorted(resp_record.values()) == sorted(record.values())
record = {"id": uuid.uuid4().hex, "name": "session4", "user": "******"}
collection_data.update(_key, json.dumps(record))
resp_record = collection_data.query_by_id(_key)
resp_record = {
key: resp_record[key]
for key in resp_record if not key.startswith("_")
}
assert sorted(resp_record.values()) == sorted(record.values())
collection_data.delete_by_id(_key)
with pytest.raises(HTTPError):
collection_data.query_by_id(_key)
def postHandler(self, authToken, queryParams):
try:
kvstore = client.Service(token=authToken,
app=APP_NAME,
autologin=True).kvstore
collection_data = kvstore['userpanel'].data
except Exception as e:
logging.error(e)
# create kvstore collection
collection_data, error_msg = self.create_collection(
kvstore, 'userpanel')
if error_msg:
payload = {'payload': {'error': error_msg}, 'status': 500}
return json.dumps(payload)
try:
params = json.loads(queryParams)
self._validate_panel_name(params.get('name'))
self._validate_panel_frequency(params.get('frequency'))
key = collection_data.insert(queryParams)
payload = {'payload': {'content': key}, 'status': 200}
return json.dumps(payload)
except Exception as e:
logging.error(e)
payload = {'payload': {'error': str(e)}, 'status': 500}
return json.dumps(payload)
def test_login_with_multiple_cookies(self):
bad_cookie = 'bad=cookie'
self.service.login()
self.assertIsNotNone(self.service.get_cookies())
service2 = client.Service(**{"cookie": bad_cookie})
# Should get an error with a bad cookie
try:
service2.login()
self.fail()
except AuthenticationError as ae:
self.assertEqual(str(ae), "Login failed.")
# Add on valid cookies, and try to use all of them
service2.get_cookies().update(self.service.get_cookies())
self.assertEqual(len(service2.get_cookies()), 2)
self.service.get_cookies().update({'bad': 'cookie'})
self.assertEqual(service2.get_cookies(), self.service.get_cookies())
self.assertEqual(len(service2.get_cookies()), 2)
self.assertTrue([cookie for cookie in service2.get_cookies() if "splunkd_" in cookie])
self.assertTrue('bad' in service2.get_cookies())
self.assertEqual(service2.get_cookies()['bad'], 'cookie')
self.assertEqual(set(self.service.get_cookies()), set(service2.get_cookies()))
service2.login()
self.assertEqual(service2.apps.get().status, 200)
def splunkrestart(host='localhost',
port=8089,
user='******',
passwd='changeme'):
"""
Restarts Splunk Services
@param host: IP or FQDN. default 'localhost'
@type host: str
@param port: splunkd service port. default 8089
@type signed_low: int
@param user: Splunk user with admin permission. default admin
@type user: str
@param passwd: Splunk user password. default 'changeme'
@type passwd: str
@rtype = dict
"""
service = client.Service(host=host,
port=port,
username=user,
password=passwd)
try:
service.login()
response = service.restart()
service.logout()
except Exception as e:
return e
return response
def connect():
global splunk
if not splunk:
app_name = os.getenv("DLTK_APP_NAME", "dltk")
base_request = base_handler()
prefix = os.getenv("SPLUNK_PATH_PREFIX", "")
def request_with_prefix(url, message, **kwargs):
o = urlparse(url)
url = "%s://%s:%s/%s%s" % (o.scheme, o.hostname, o.port, prefix,
o.path)
if o.query:
url += "?" + o.query
return base_request(url, message, **kwargs)
splunk = client.Service(
handler=request_with_prefix if prefix else None,
username=os.getenv("SPLUNK_USERNAME", "admin"),
password=os.getenv("SPLUNK_PASSWORD", "changeme"),
host=os.getenv("SPLUNK_HOST", "localhost"),
scheme=os.getenv("SPLUNK_SCHEME", "https"),
port=int(os.getenv("SPLUNK_PORT", "8089")),
sharing="app",
app=app_name,
autologin=True,
)
return splunk
def test_login_with_multiple_cookies(self):
bad_cookie = 'bad=cookie'
self.service.login()
self.assertIsNotNone(self.service.get_cookies())
service2 = client.Service(**{"cookie": bad_cookie})
service2.login()
# Should get an error with a bad cookie
try:
service2.apps.get()
self.fail()
except AuthenticationError as ae:
self.assertEqual(ae.message,
"Request failed: Session is not logged in.")
# Add on valid cookies, and try to use all of them
service2.get_cookies().update(self.service.get_cookies())
self.assertEqual(len(service2.get_cookies()), 2)
self.service.get_cookies().update({'bad': 'cookie'})
self.assertEqual(service2.get_cookies(),
self.service.get_cookies())
self.assertEqual(len(service2.get_cookies()), 2)
self.assertEqual(service2.get_cookies().keys()[1][:8], "splunkd_")
self.assertTrue('bad' in service2.get_cookies().keys())
self.assertEqual(service2.get_cookies()['bad'], 'cookie')
self.assertEqual(self.service.get_cookies().items(),
service2.get_cookies().items())
service2.login()
self.assertEqual(service2.apps.get().status, 200)
def create_service(self):
s = client.Service(
#token=self.sessionKey,
token=self.request["systemAuth"],
sharing="app",
app=app_name,
)
return s
def _validate(self, props):
c = client.Service(**props)
# check whether it installed aws add-on
if props['host'] is not self.local_splunk_host:
c.login()
if TARGET_APP not in c.apps:
return False
return True
def splunk(self):
if self._splunk != None:
return self._splunk
self._splunk = client.Service(
token=self.request["systemAuth"], # self.sessionKey,
sharing="app",
app=this_app_name,
)
return self._splunk
def setUp(self):
self.opts = testlib.parse([], {}, ".splunkrc")
self.service = client.Service(**self.opts.kwargs)
# Skip these tests if running below Splunk 6.2, cookie-auth didn't exist before
splver = self.service.splunk_version
if splver[:2] < (6, 2):
self.skipTest(
"Skipping cookie-auth tests, running in %d.%d.%d, this feature was added in 6.2+"
% splver)
def test_login_fails_with_no_cookie(self):
service2 = client.Service()
self.assertEquals(len(service2.get_cookies()), 0)
# Should get an error when no authentication method
try:
service2.login()
self.fail()
except AuthenticationError as ae:
self.assertEqual(ae.message, "Login failed.")
def setUp(self):
self.opts = testlib.parse([], {}, ".splunkrc")
opts = self.opts.kwargs.copy()
opts["basic"] = True
opts["username"] = self.opts.kwargs["username"]
opts["password"] = self.opts.kwargs["password"]
self.context = binding.connect(**opts)
import splunklib.client as client
service = client.Service(**opts)
def test_login_with_cookie(self):
self.service.login()
self.assertIsNotNone(self.service.get_cookies())
# Use the cookie from the other service as the only auth param (don't need user/password)
service2 = client.Service(**{"cookie": "%s=%s" % list(self.service.get_cookies().items())[0]})
service2.login()
self.assertEqual(len(service2.get_cookies()), 1)
self.assertEqual(service2.get_cookies(), self.service.get_cookies())
self.assertEqual(len(service2.get_cookies()), len(self.service.get_cookies()))
self.assertEqual(list(service2.get_cookies().keys())[0][:8], "splunkd_")
self.assertEqual(service2.apps.get().status, 200)
def setUp(self):
self.opts = testlib.parse([], {}, ".splunkrc")
self.service = client.Service(**self.opts.kwargs)
# Skip these tests if running below Splunk 6.2, cookie-auth didn't exist before
splver = self.service.splunk_version
# TODO: Workaround the fact that skipTest is not defined by unittest2.TestCase
if splver[:2] < (6, 2):
self.skipTest(
"Skipping cookie-auth tests, running in %d.%d.%d, this feature was added in 6.2+"
% splver)
def test_login_fails_with_bad_cookie(self):
bad_cookie = {'bad': 'cookie'}
service2 = client.Service()
self.assertEqual(len(service2.get_cookies()), 0)
service2.get_cookies().update(bad_cookie)
self.assertEqual(service2.get_cookies(), {'bad': 'cookie'})
# Should get an error with a bad cookie
try:
service2.login()
self.fail()
except AuthenticationError as ae:
self.assertEqual(str(ae), "Login failed.")
def test_submit_via_attach_with_multiple_cookie_headers(self):
# Skip this test if running below Splunk 6.2, cookie-auth didn't exist before
splver = self.service.splunk_version
if splver[:2] < (6, 2):
self.skipTest("Skipping cookie-auth tests, running in %d.%d.%d, this feature was added in 6.2+" % splver)
event_count = int(self.service.indexes[self.index_name]['totalEventCount'])
service = client.Service(**{"cookie": 'a bad cookie'})
service.http._cookies.update(self.service.http._cookies)
service.login()
cn = service.indexes[self.index_name].attach()
cn.send(b"Hello Boris!\r\n")
cn.close()
self.assertEventuallyTrue(lambda: self.totalEventCount() == event_count+1, timeout=60)
def connect():
global splunk
if not splunk:
app_name = os.getenv("DLTK_APP_NAME", "dltk")
splunk = client.Service(
username=os.getenv("SPLUNK_USERNAME", "admin"),
password=os.getenv("SPLUNK_PASSWORD", "changeme"),
host=os.getenv("SPLUNK_HOST", "localhost"),
port=int(os.getenv("SPLUNK_PORT", "8089")),
sharing="app",
app=app_name,
autologin=True,
)
return splunk
def test_login_fails_with_bad_cookie(self):
bad_cookie = {'bad': 'cookie'}
service2 = client.Service()
self.assertEquals(len(service2.get_cookies()), 0)
service2.get_cookies().update(bad_cookie)
service2.login()
self.assertEquals(service2.get_cookies(), {'bad': 'cookie'})
# Should get an error with a bad cookie
try:
service2.apps.get()
self.fail()
except AuthenticationError as ae:
self.assertEqual(ae.message,
"Request failed: Session is not logged in.")
def deleteHandler(self, authToken, query):
try:
kvstore = client.Service(token=authToken,
app=APP_NAME,
autologin=True).kvstore
collection_data = kvstore['userpanel'].data
key = self.parseGetParams(query)
queryString = json.dumps({'_key': key})
collection_data.delete(query=queryString)
payload = {'payload': {'content': {'key': key}}, 'status': 200}
return json.dumps(payload)
except Exception as e:
logging.error(e)
payload = {'payload': {'error': str(e)}, 'status': 500}
return json.dumps(payload)
def putHandler(self, authToken, payload):
try:
kvstore = client.Service(token=authToken,
app=APP_NAME,
autologin=True).kvstore
collection_data = kvstore['userpanel'].data
params = json.loads(payload)
self._validate_panel_name(params.get('name'))
self._validate_panel_frequency(params.get('frequency'))
key = params.get('key')
collection_data.update(key, payload)
payload = {'payload': {'content': {'key': key}}, 'status': 200}
return json.dumps(payload)
except Exception as e:
logging.error(e)
payload = {'payload': {'error': str(e)}, 'status': 500}
return json.dumps(payload)
def _get_svc(self):
host, port = entity.getEntity(
'configs/conf-web',
'settings',
sessionKey = self.sessionKey,
namespace = self.context['app'],
owner = self.context['user']
).get(
'mgmtHostPort', DEFAULT_HOST_PORT
).split(':')
return client.Service(
host = host,
port = port,
app = self.context['app'],
owner = self.context['user'],
token = self.sessionKey
)
def getHandler(self, authToken, query):
try:
kvstore = client.Service(token=authToken,
app=APP_NAME,
autologin=True).kvstore
collection_data = kvstore['userpanel'].data.query()
payload = {
'payload': {
'content': {
'entry': collection_data
}
},
'status': 200
}
return json.dumps(payload)
except Exception as e:
logging.error(e)
payload = {'payload': {'error': str(e)}, 'status': 500}
return json.dumps(payload)
def __init__(self,
app=None,
owner='nobody',
session_key=None,
service=None):
self._app = app
self._owner = owner
self._session_key = session_key
if service is None:
splunkd_host_port = self._get_entity(CONF_WEB, 'settings').get(
'mgmtHostPort', '127.0.0.1:8089')
host_and_port = splunkd_host_port.split(':')
self.local_splunk_host = host_and_port[0]
self.local_splunk_port = host_and_port[1]
self._service = client.Service(host=self.local_splunk_host,
port=self.local_splunk_port,
app=self._app,
owner=self._owner,
token=self._session_key)
else:
self._service = service
def _get_target_service(self, target, target_app=None, target_owner=None):
logger.info('get target service %s' % target)
target_manager = tm.TargetManager(app=self.appName,
owner=self.userName,
session_key=self.getSessionKey())
target_properties = target_manager.get_target(target)
if target_app:
target_properties['app'] = target_app
if target_owner:
target_properties['owner'] = target_owner
if target_properties:
logger.info('target service props %s' % target_properties)
service = client.Service(**target_properties)
# TODO: anyway to cache the session key?
if not target == target_manager.local_splunk_host:
service = service.login()
logger.info('login %s ! %s' % (service.host, service.token))
return service
else:
raise admin.ArgValidationException("target %s does not exist" %
target)
def putHandler(self, authToken, in_string):
try:
confs = client.Service(token=authToken,
app=APP_NAME,
autologin=True).confs
macros = confs['macros']
data = json.loads(in_string)
jenkins = data['jenkins']
statistics = data['jenkinsStatistics']
console = data['jenkinsConsole']
self._validate_user_input(jenkins)
self._validate_user_input(statistics)
self._validate_user_input(console)
for stanza in macros:
if stanza.name == 'jenkins_statistics_index':
stanza.update(**{'definition': 'index=' + statistics})
elif stanza.name == 'jenkins_console_index':
stanza.update(**{'definition': 'index=' + console})
elif stanza.name == 'jenkins_index':
stanza.update(**{'definition': 'index=' + jenkins})
payload = {
'payload': {
'content': {
'entry': {
'statistics': statistics,
'jenkins': jenkins,
'consoleIdx': console
}
}
},
'status': 200
}
return json.dumps(payload)
except Exception as e:
logging.error(str(e))
payload = {'payload': {'error': str(e)}, 'status': 500}
return json.dumps(payload)
def __init__(self, app=None, owner=None, session_key=None):
self._app = app
self._owner = 'nobody' # so that conf file will be saved in app
self._sessionKey = session_key
splunkd_host_port = self._get_entity(CONF_WEB, 'settings').get(
'mgmtHostPort', '127.0.0.1:8089')
host_and_port = splunkd_host_port.split(':')
self.local_splunk_host = host_and_port[0]
self.local_splunk_port = host_and_port[1]
logger.info('app %s, owner %s, host %s, port %s' %
(self._app, self._owner, self.local_splunk_host,
self.local_splunk_port))
self._service = client.Service(host=self.local_splunk_host,
port=self.local_splunk_port,
app=self._app,
owner=self._owner,
token=self._sessionKey)
if "Splunk_TA_snow" not in self._service.apps:
raise admin.InternalException(
"Splunk ServiceNow Add-on is not found on server")
if CONF_TARGET not in self._service.confs:
self._service.confs.create(CONF_TARGET)
def getHandler(self, authToken, params):
try:
confs = client.Service(token=authToken,
app=APP_NAME,
autologin=True).confs
macros = confs['macros']
for stanza in macros:
if stanza.name == 'jenkins_statistics_index':
statistics = stanza.content
elif stanza.name == 'jenkins_console_index':
console = stanza.content
elif stanza.name == 'jenkins_index':
jenkins = stanza.content
payload = {
'payload': {
'content': {
'entry': {
'statistics': statistics,
'consoleIdx': console,
'jenkins': jenkins
}
}
},
'status': 200
}
return json.dumps(payload)
except Exception as e:
logging.error(str(e))
payload = {
'payload': {
'error': str(e),
},
'status': 500
}
return json.dumps(payload)
def post(self, **kwargs):
app_name = "%s" % cherrypy.request.path_info.split("/")[-3]
try:
lib_path = sys.path.append(
make_splunkhome_path(
["etc", "apps", "splunk_as_a_service", "lib"]))
if lib_path not in sys.path:
sys.path.insert(0, lib_path)
bin_path = sys.path.append(
make_splunkhome_path(
["etc", "apps", "splunk_as_a_service", "bin"]))
if bin_path not in sys.path:
sys.path.insert(0, bin_path)
import capabilities
import apps
import app_bundles
import importlib
importlib.reload(capabilities)
importlib.reload(apps)
importlib.reload(app_bundles)
if not capabilities.has(
"saas_manage_apps") and not capabilities.has(
"admin_all_objects"):
cherrypy.response.status = 403
return "missing capability"
app_field = kwargs.get('app')
# if not isinstance(app_field, cgi.FieldStorage):
# cherrypy.response.status = 400
# return "Missing app field: %s" % dir(app_field)
import splunklib.client as client
splunk = client.Service(
token=cherrypy.session.get('sessionKey'),
sharing="app",
app=app_name,
)
app_path = None
try:
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
shutil.copyfileobj(app_field.file, temp_file)
app_path = temp_file.name
if app_bundles.is_bundle(app_path):
bundle_name = app_bundles.add_bundle(
splunk, app_path, app_field.filename)
return self.render_json(
dict(
kind="bundle",
name=bundle_name,
))
else:
app_name, app_version = apps.add_app(splunk, app_path)
return self.render_json(
dict(kind="app", name=app_name, version=app_version))
finally:
if app_path:
os.remove(app_path)
except:
cherrypy.response.status = 500
return traceback.format_exc()
def generate(self):
self.logger.debug("Generating %s events" % self.incident_id)
service = client.Service(token=self.metadata.searchinfo.session_key)
# Check if configuration exists for collect_data_results
try:
collect_data_results = service.confs['alert_manager']['settings'][
'collect_data_results']
except:
raise RuntimeWarning(
'Specified setting ""collect_data_results" in "alert_manager.conf" does not exist.'
)
# Check if configuration exists for index_data_results
try:
index_data_results = service.confs['alert_manager']['settings'][
'index_data_results']
except:
raise RuntimeWarning(
'Specified setting ""index_data_results" in "alert_manager.conf" does not exist.'
)
# Fetch Results from KV Store by default if enabled
if collect_data_results == '1':
service.namespace['owner'] = "Nobody"
collection_name = "incident_results"
collection = service.kvstore[collection_name]
query_dict = {}
query_dict['incident_id'] = self.incident_id
query = json.dumps(query_dict)
data = collection.data.query(query=query)
incident_data = {}
incident_data = data[0].get("fields")[0]
for k, v in incident_data.items():
incident_data[k] = urllib.parse.quote(v)
# If KV Store Data is not enabled, get indexed data
elif index_data_results == '1' and collect_data_results == '0':
# Get index location
try:
index = service.confs['alert_manager']['settings']['index']
except:
raise RuntimeWarning(
'Specified setting ""index_data_results" in "alert_manager.conf" does not exist.'
)
# Get earliest time first for incident results
service.namespace['owner'] = "Nobody"
collection_name = "incidents"
collection = service.kvstore[collection_name]
query_dict = {}
query_dict['incident_id'] = self.incident_id
query = json.dumps(query_dict)
data = collection.data.query(query=query)
earliest_time = data[0].get("alert_time")
# Fetch events
events = []
incident_data = {}
kwargs_oneshot = json.loads(
'{{"earliest_time": "{}", "latest_time": "{}"}}'.format(
earliest_time, "now"))
searchquery_oneshot = "search index={} sourcetype=alert_data_results incident_id={} |dedup incident_id".format(
index, self.incident_id)
oneshotsearch_results = service.jobs.oneshot(
searchquery_oneshot, **kwargs_oneshot)
reader = results.ResultsReader(oneshotsearch_results)
for result in reader:
for k, v in result.items():
if k == '_raw':
events.append(json.loads(v))
for event in events:
incident_data = event.get("fields")[0]
for k, v in incident_data.items():
incident_data[k] = urllib.parse.quote(v)
# Get Incident
query_dict = {}
query_dict['incident_id'] = self.incident_id
collection_name = "incidents"
collection = service.kvstore[collection_name]
query = json.dumps(query_dict)
data = collection.data.query(query=query)
alert = data[0].get('alert')
# Get Incident Settings
query_dict = {}
query_dict['alert'] = alert
collection_name = "incident_settings"
collection = service.kvstore[collection_name]
query = json.dumps(query_dict)
data = collection.data.query(query=query)
drilldown_references = data[0].get('drilldowns')
# Get Drilldown Settings
if len(drilldown_references) > 0:
query_dict = {}
drilldown_references = drilldown_references.split()
query_prefix = '{ "$or": [ '
for drilldown_reference in drilldown_references:
query_prefix += '{ "name": "' + drilldown_reference + '" } '
query_prefix = query_prefix + '] }'
collection_name = "drilldown_actions"
collection = service.kvstore[collection_name]
query = query_prefix.replace("} {", "}, {")
data = collection.data.query(query=query)
drilldowns = []
# Substitute variables with field values
for drilldown_action in data:
url = drilldown_action.get("url")
label = drilldown_action.get("label")
url = re.sub(r'(?<=\w)\$', '', url)
class FieldTemplate(StringTemplate):
idpattern = r'[a-zA-Z][_a-zA-Z0-9.]*'
url_template = FieldTemplate(url)
url = url_template.safe_substitute(incident_data)
drilldown = r'''{{ "label": "{}", "url": "{}" }}'''.format(
label, url)
yield (json.loads(drilldown))
