class QueryRunner(object):
""" Query Runner.
a class to handle query to splunkd.
It grabs a splunkd object according to the splunkrc params provided:
- If splunkrc is a dictionary, it will create a new splunkd object.
- If given other object type, it will do do Dependency Injection on _splunkd
"""
def __init__(self,
splunkrc,
index_name=INSTRUMENTATION_INDEX_NAME,
owner='-',
app=INST_APP_NAME,
result_reader=QueryRunnerResult):
self.splunkrc = splunkrc or SPLUNKRC
self.result_reader = result_reader
if type(self.splunkrc) is dict:
self._splunkd = Splunkd(**self.splunkrc)
else:
self._splunkd = splunkrc
self._splunkd.namespace['owner'] = owner
self._splunkd.namespace['app'] = app
if self._splunkd.has_index(index_name):
self._index = self._splunkd.get_index(index_name)
else:
logging.error('ERROR: INDEX IS NOT AVAILABLE')
raise (Exception("ERROR INDEX UNAVAILABLE"))
def search(self, search_cmd, **kwargs):
"""Submit a new search.
It is a wrapper to the private method _query.
"""
return self._query(search_cmd, **kwargs)
def _query(self, search_cmd, **kwargs):
"""Query.
Note that earliest is inclusive & latest is exclusive:
[earliest, latest)
(Prevents getting last-second events again during the next query)
"""
job = self._splunkd.search(search_cmd, **kwargs)
while not job.is_done():
time.sleep(0.2)
result = self.result_reader(job.results(count=0))
if hasattr(result, 'job'):
result.job = job
return result
def __init__(self,
splunkd,
telemetry_conf_service=None,
server_info_service=None):
self.splunkd = Splunkd.decorate(splunkd)
self._telemetry_conf_service = telemetry_conf_service
self._server_info_service = server_info_service
def __init__(self, splunkrc=None, index_name=None):
self.splunkrc = splunkrc or SPLUNKRC
self.socket = None
self._index = None
if type(self.splunkrc) is dict:
self._splunkd = Splunkd(**self.splunkrc)
else:
self._splunkd = splunkrc
if index_name:
if self._splunkd.has_index(index_name):
self._index = self._splunkd.get_index(index_name)
else:
logging.error('ERROR: INDEX IS NOT AVAILABLE')
raise Exception("ERROR INDEX UNAVAILABLE")
def instrumentation_eligibility(self, optInVersion=None, **kwargs):
'''
Determines whether the UI for the instrumentation app should be visible,
including the initial opt-in modal and all settings/logs pages.
This is determined by user capabilities, license type, and server roles.
'''
cherrypy.response.headers['Content-Type'] = 'application/json'
services = ServiceBundle(Splunkd(**self.splunkrc()))
currentOptInVersion = services.telemetry_conf_service.content.get('optInVersion')
if optInVersion != '*' and optInVersion != currentOptInVersion:
return json.dumps({
'is_eligible': False,
'reason': 'UNAUTHORIZED',
})
# If we're not running on a free license (where there are no users),
# first validate that the user has the requisite capabilities.
if services.server_info_service.content.get('isFree', '0') != '1':
user = en.getEntity('authentication/users', cherrypy.session['user']['name'])
if 'edit_telemetry_settings' not in user.properties['capabilities']:
return json.dumps({
'is_eligible': False,
'reason': 'UNAUTHORIZED'
})
# Now check the server roles, etc
eligibility = client_eligibility.get_ui_eligibility(services)
return json.dumps(eligibility)
def main():
sleep(INST_PRE_EXECUTE_SLEEP)
if os.environ['INST_MODE'] == "DEV":
splunkd = Splunkd(**SPLUNKRC)
else:
splunkd = Splunkd(token=os.environ['INST_TOKEN'], server_uri=SPLUNKD_URI)
services = ServiceBundle(splunkd)
telemetry_conf_service = services.telemetry_conf_service
if should_input_run(telemetry_conf_service):
process_input_params(telemetry_conf_service, args)
run_input({'start': args.start_date, 'stop': args.stop_date})
else:
# indicate to caller that input wasn't executed
sys.exit(114)
def __connect_to_splunkd(self, search=False):
if search:
# WITH NAMESPACE
splunkrc_copy = {key: self.splunkrc[key] for key in self.splunkrc}
if 'owner' not in splunkrc_copy:
splunkrc_copy['owner'] = '-'
if 'app' not in splunkrc_copy:
splunkrc_copy['app'] = INST_APP_NAME
return Splunkd(**splunkrc_copy)
else:
splunkrc_copy = {key: self.splunkrc[key] for key in self.splunkrc}
if 'owner' in splunkrc_copy:
splunkrc_copy.pop('owner')
if 'app' in splunkrc_copy:
splunkrc_copy.pop('app')
return Splunkd(**self.splunkrc)
def __init__(self, metricSchema, dataPointFactory, splunkrc=None, phase=1):
self.metricSchema = metricSchema
self.dataPointFactory = dataPointFactory
self.splunkrc = splunkrc
self.profile = get_instance_profile()
self.phase = phase
splunkd = Splunkd(**SPLUNKRC)
services = ServiceBundle(splunkd)
salt_manager = SaltManager(services)
self.salt = salt_manager.get_salt()
self.scheme = {"hash": self.salt}
def __init__(self,
splunkrc,
index_name=INSTRUMENTATION_INDEX_NAME,
owner='-',
app=INST_APP_NAME,
result_reader=QueryRunnerResult):
self.splunkrc = splunkrc or SPLUNKRC
self.result_reader = result_reader
if type(self.splunkrc) is dict:
self._splunkd = Splunkd(**self.splunkrc)
else:
self._splunkd = splunkrc
self._splunkd.namespace['owner'] = owner
self._splunkd.namespace['app'] = app
if self._splunkd.has_index(index_name):
self._index = self._splunkd.get_index(index_name)
else:
logging.error('ERROR: INDEX IS NOT AVAILABLE')
raise (Exception("ERROR INDEX UNAVAILABLE"))
def getDeploymentID(self, cookie):
if not self.deploymentID:
self.getToken(cookie)
self.server_uri = rest.makeSplunkdUri()
splunkd = Splunkd(token=self.token, server_uri=self.server_uri)
telemetry_conf_service = TelemetryConfService(splunkd,
is_read_only=True)
telemetry_conf_service.fetch()
deployment_id_manager = DeploymentIdManager(
splunkd, telemetry_conf_service=telemetry_conf_service)
self.deploymentID = deployment_id_manager.get_deployment_id()
return self.deploymentID
parser.add_argument('-p', '--password',
help='Splunk password',
required=True)
parser.add_argument('--prefix',
help='Desired prefix for the deployment ID',
required=False)
args = parser.parse_args()
splunkrc = {
'username': args.user,
'password': args.password
}
splunkd = Splunkd(**splunkrc)
deployment_id_manager = DeploymentIdManager(splunkd)
deploymentID = deployment_id_manager.get_deployment_id(no_create=True)
if deploymentID is not None and deploymentID.startswith(args.prefix or ''):
print("Deployment ID already initialized: %s" %
deploymentID)
# Only failures to set are considered error conditions.
# So the exit code for an existing deployment ID is still 0.
exit(0)
else:
deployment_id_manager.generate_new_deployment_id(prefix=args.prefix)
deployment_id_manager.write_deployment_id_to_conf_file()
print("Deployment ID successfully initialized: %s" %
deployment_id_manager.get_deployment_id(no_create=True))
['content']['disabled'])
except Exception:
# Proceed as if it's disabled if we can't hit the endpoint
pass
if dist_search_disabled:
return True
search_peers = splunkd.get_json(
'/services/search/distributed/peers')['entry']
if search_peers:
for peer in search_peers:
if (peer['content']['status'].lower() == 'up'
and not peer['content']['disabled'] and '_telemetry'
not in peer['content']['searchable_indexes']):
return False
return True
if __name__ == '__main__':
from splunk_instrumentation.splunkd import Splunkd
from splunk_instrumentation.service_bundle import ServiceBundle
from splunk_instrumentation.cli_token import get_token
bundle = ServiceBundle(Splunkd(token=get_token('https://localhost:8089')))
from pprint import pprint
pprint(get_eligibility(bundle, opt_in_version='*'))
def get_service(self, token, service=None):
self.token = token
self.server_uri = self.make_splunkd_uri()
if not service:
service = Splunkd(token=self.token, server_uri=self.server_uri)
return service
import sys
import time
import splunk_instrumentation.splunklib as splunklib
from splunk_instrumentation.splunkd import Splunkd
from splunk_instrumentation.service_bundle import ServiceBundle
from splunk_instrumentation.deployment_id_manager import DeploymentIdManager
import splunk_instrumentation.constants as constants
import splunk_instrumentation.client_eligibility as client_eligibility
from splunk_instrumentation.salt_manager import SaltManager
token = sys.stdin.read().rstrip()
splunkd = Splunkd(token=token, server_uri=constants.SPLUNKD_URI)
services = ServiceBundle(splunkd)
salt_manager = SaltManager(services)
# Cloud provisioning sets their own deployment
# ID's after starting splunk. But just in case
# we're stuck with the value from this script,
# let's use the correct prefix.
prefix = None
if services.server_info_service.is_cloud():
prefix = 'CLOUD'
if services.server_info_service.is_lite():
prefix += 'LIGHT'
# Migration of the deployment ID from V1 of instrumentation
# requires waiting until the KV store is ready. We'll give
# it 5 minutes, then proceed without out.
t_start = time.time()
splunk cmd python -m splunk_instrumentation.deployment_node_list
# Alternatively, with a remote splunk
splunk cmd python -m splunk_instrumentation.deployment_node_list https://remote_splunk:8089
Example:
$ splunk cmd python -m splunk_instrumentation.deployment_node_list
{'errors': ['Could not locate any search head cluster members in this deployment',
'Failed to locate a Cluster Master in this deployment'],
'nodes': [{'authMethod': 'indexer',
'host': u'9ac296fad4e8',
'roles': [u'indexer', u'license_master', u'search_peer'],
'uri': 'https://localhost:8090'}]}
'''
import sys
from splunk_instrumentation.splunkd import Splunkd
from splunk_instrumentation.cli_token import get_token
from pprint import pprint
if len(sys.argv) > 1:
splunk_uri = sys.argv[1]
else:
splunk_uri = 'https://localhost:8089'
service = Splunkd(server_uri=splunk_uri, token=get_token(splunk_uri))
node_list = NodeList(service)
pprint(node_list.fetch_nodes())
def update(self,
cherrypy,
services=None,
deployment_id_manager=None,
salt_manager=None):
'''
Updates the volatile data members of the swa context.
This method is hit each time an HTML page is hit, so the
less work done here the better.
'''
if services is None:
splunkd = Splunkd(token=cherrypy.session.get('sessionKey'),
server_uri=self.server_uri)
telemetry_conf_service = TelemetryConfService(splunkd,
is_read_only=True)
telemetry_conf_service.fetch()
# Specialize the telemetry_conf_service to be read only up front,
# use the default construction for other services.
services = ServiceBundle(
splunkd, telemetry_conf_service=telemetry_conf_service)
if not self.instance_guid:
self.instance_guid = services.server_info_service.content.get(
'guid')
salt_manager = salt_manager or SaltManager(services)
self.salt = salt_manager.get_salt()
deployment_id_manager = deployment_id_manager or DeploymentIdManager(
splunkd, telemetry_conf_service=services.telemetry_conf_service)
self.deployment_id = deployment_id_manager.get_deployment_id() or ''
self.opt_in_is_up_to_date = services.telemetry_conf_service.opt_in_is_up_to_date(
)
self.swa_base_url = services.telemetry_conf_service.content.get(
'swaEndpoint')
hash_key = self.salt + splunk.auth.getCurrentUser()['name']
if sys.version_info >= (3, 0):
hash_key = hash_key.encode()
self.user_id = hashlib.sha256(hash_key).hexdigest()
self.send_anonymized_web_analytics = conf_bool(
services.telemetry_conf_service.content.get(
'sendAnonymizedWebAnalytics'))
visibilities = []
if services.server_info_service.is_cloud():
visibilities = ['anonymous', 'support']
else:
if conf_bool(
services.telemetry_conf_service.content.get(
'sendAnonymizedUsage')):
visibilities.append('anonymous')
if conf_bool(
services.telemetry_conf_service.content.get(
'sendSupportUsage')):
visibilities.append('support')
self.visibility = ','.join(visibilities)
if self.send_anonymized_web_analytics and not self.cds_url:
self.cds_url = get_quick_draw().get('url')
class EventWriter(object):
""" Event Writer class
This class handles writing to the index.
It grabs a splunkd object according to the splunkrc params provided:
- If splunkrc is a dictionary, it will create a new splunkd object.
- If given other object type, it will do do Dependency Injection on _splunkd
"""
def __init__(self, splunkrc=None, index_name=None):
self.splunkrc = splunkrc or SPLUNKRC
self.socket = None
self._index = None
if type(self.splunkrc) is dict:
self._splunkd = Splunkd(**self.splunkrc)
else:
self._splunkd = splunkrc
if index_name:
if self._splunkd.has_index(index_name):
self._index = self._splunkd.get_index(index_name)
else:
logging.error('ERROR: INDEX IS NOT AVAILABLE')
raise Exception("ERROR INDEX UNAVAILABLE")
def submit(self, event, host=None, source=None, sourcetype=INSTRUMENTATION_SOURCETYPE):
# Note: We used to use the ordinary index.submit method from splunklib,
# instead of using a socket. However, that method uses receivers/simple,
# an endpoint that bypasses index time field extraction (which we rely on).
temp_socket = self._index.attach(host=host, source=source, sourcetype=sourcetype)
temp_socket.send(self.marshal_event(event))
temp_socket.close()
def open_socket(self, host=None, source=None, sourcetype=INSTRUMENTATION_SOURCETYPE):
'''
Opens a socket to stream events to be indexed, saving it as
an instance variable for later use when submit_via_socket is called.
:param host:
:param source:
:param sourcetype:
:return:
'''
self.socket = self._index.attach(host=host, source=source, sourcetype=sourcetype)
return self.socket
def close_socket(self):
'''
Closes socket and set it to none
'''
if self.socket:
self.socket.close()
self.socket = None
def submit_via_socket(self, event):
"""
Submit the event provided using socket connection.
"""
event = self.marshal_event(event)
if not self.socket:
self.open_socket()
self.socket.send(event)
@staticmethod
def marshal_event(event):
'''
Marshals the given event into a json string, suitable for passing
to an open receivers/stream socket.
'''
if not isinstance(event, (str, bytes)):
event = json.dumps(event, default=json_serial)
if isinstance(event, str) and sys.version_info >= (3, 0):
event = event.encode()
return event
"""Spawn multiple workers and wait for them to complete"""
path = os.path.realpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', '..',
'bin'))
sys.path.append(path)
from splunk_instrumentation.diag.batch_runner import BatchRunner # noqa
from splunk_instrumentation.diag.diag_task import DiagTask # noqa
from splunk_instrumentation.splunkd import Splunkd # noqa
from splunk_instrumentation.diag.diag_service import DiagService # noqa
configurationStr = sys.stdin.read()
configuration = json.loads(configurationStr)
splunkd = Splunkd(token=configuration['token'],
server_uri=configuration['server_uri'])
diag_service = DiagService(splunkd)
batchRunner = BatchRunner(config=configuration['payload'])
diag_service.batch_id = batchRunner.batch_id
for config in configuration['payload']['nodes']:
batchRunner.add_task(
DiagTask(
{
"node": config,
"configuration": configuration['payload']['configuration']
},
diag_service=diag_service))
def handle(self, arg):
'''
Takes the parsed request data passed by splunkd to
PersistentServerConnectionApplication.handle and returns a response.
:param arg: JSON object
:return: JSON object
'''
arg = self.parse_arg(arg)
logger.debug('arg = %s' % json.dumps(arg))
if 'query' not in arg:
arg['query'] = []
try:
if 'session' not in arg:
raise splunk.RESTException(500, "No session found.")
self.session = arg['session']
if 'system_authtoken' not in arg:
raise splunk.RESTException(500, "No system auth token found.")
self.system_authtoken = arg['system_authtoken']
if self.services:
self.splunkd = self.services.splunkd
else:
self.splunkd = Splunkd(**self.splunkrc())
self.services = ServiceBundle(self.splunkd)
if not self.system_services:
splunkd = Splunkd(**self.system_splunkrc())
self.system_services = ServiceBundle(splunkd)
usage_data_endpoint_table = {
'anonymous_usage_data': {
'visibility': 'anonymous',
'action': 'export'
},
'license_usage_data': {
'visibility': 'license',
'action': 'export'
},
'support_usage_data': {
'visibility': 'support',
'action': 'export'
},
'send_anonymous_usage_data': {
'visibility': 'anonymous',
'action': 'send'
},
'send_license_usage_data': {
'visibility': 'license',
'action': 'send'
},
'send_support_usage_data': {
'visibility': 'support',
'action': 'send'
}
}
path = arg['path_info']
if path == 'instrumentation_eligibility':
return self.response_to_eligibility_request(arg)
elif path in usage_data_endpoint_table:
visibility = usage_data_endpoint_table[path]['visibility']
if (usage_data_endpoint_table[path]['action'] == 'export'):
return self.response_to_export_request(
path, visibility, arg)
else:
return self.response_to_send_request(path, visibility, arg)
else:
return {'payload': '"%s" not found' % path, 'status': 404}
except splunk.RESTException as e:
logger.error(e)
return {
'payload': 'Exception caught: %s' % e.msg,
'status': e.statusCode
}
except Exception as e:
logger.error('ERROR: ' + traceback.format_exc())
return {
'payload': traceback.format_exception_only(type(e), e)[-1],
'status': 500
}
class EventWriter(object):
""" Event Writer class
This class handles writing to the index.
It grabs a splunkd object according to the splunkrc params provided:
- If splunkrc is a dictionary, it will create a new splunkd object.
- If given other object type, it will do do Dependency Injection on _splunkd
"""
def __init__(self, splunkrc=None, index_name=None):
self.splunkrc = splunkrc or SPLUNKRC
self.socket = None
self._index = None
if type(self.splunkrc) is dict:
self._splunkd = Splunkd(**self.splunkrc)
else:
self._splunkd = splunkrc
if index_name:
if self._splunkd.has_index(index_name):
self._index = self._splunkd.get_index(index_name)
else:
logging.error('ERROR: INDEX IS NOT AVAILABLE')
raise (Exception("ERROR INDEX UNAVAILABLE"))
def submit(self, event, host=None, source=None, sourcetype=None):
"""Submit a new event directly to the index."""
if self._index:
self._index.submit(event,
host=host,
source=source,
sourcetype=sourcetype)
else:
logging.error('ERROR: INDEX IS NOT AVAILABLE')
raise Exception("ERROR INDEX UNAVAILABLE")
def open_socket(self, host=None, source=None, sourcetype=None):
'''
Opens a socket to stream events to be indexed
:param host:
:param source:
:return:
'''
self.socket = self._index.attach(host=host,
source=source,
sourcetype=INSTRUMENTATION_SOURCETYPE)
return self.socket
def close_socket(self):
'''
Closes socket and set it to none
'''
if self.socket:
self.socket.close()
self.socket = None
def submit_via_socket(self, event):
"""
Submit the event provided using socket connection.
"""
if not isinstance(event, str):
event = json.dumps(event, default=json_serial)
if not self.socket:
self.open_socket()
self.socket.send(event)