def clean(self, endpoint):
url = self.make_uri(endpoint) + '?count=-1'
resp, cont = splunkd_request(url,
self.getSessionKey(),
method='GET',
data={'output_mode': 'json'},
retry=3)
if resp.status != 200:
return {url: self.convertErrMsg(cont)}
cont = json.loads(cont)
ents = [ent['name'] for ent in cont.get('entry', [])]
errs = {}
for ent in ents:
args = self.getArgs()
if not self.distinguish(endpoint, ent, **args):
continue
url_ent = self.make_uri(endpoint, entry=ent)
resp, cont = splunkd_request(url_ent,
self.getSessionKey(),
method='DELETE',
data={'output_mode': 'json'},
retry=3)
if resp.status != 200:
errs[url_ent] = self.convertErrMsg(cont)
return errs
def reload_http_input(self):
"""
:return: Response object and content
"""
uri = "{}/servicesNS/-/{}/data/inputs/http/_reload?output_mode=json".format(
self._conf_mgr.splunkd_uri, "splunk_httpinput")
sr.splunkd_request(uri, self._conf_mgr.session_key)
def user_caps(mgmt_uri, session_key):
"""
Get capabilities of sessioned Splunk user.
:param mgmt_uri:
:param session_key:
:return:
"""
url = mgmt_uri.strip('/') + '/services/authentication/current-context'
resp, cont = splunkd_request(url,
session_key,
method='GET',
data={'output_mode': 'json'},
retry=3
)
if resp is None:
RH_Err.ctl(500,
logLevel=logging.ERROR,
msgx='Fail to get capabilities of sessioned user')
elif resp.status not in (200, '200'):
RH_Err.ctl(resp.status,
logLevel=logging.ERROR,
msgx=cont
)
cont = json.loads(cont)
caps = cont['entry'][0]['content']['capabilities']
return set(caps)
def _do_request(self,
uri,
method,
data=None,
content_type="application/x-www-form-urlencoded"):
headers = {"Content-Type": content_type}
resp, content = rest.splunkd_request(uri,
self._session_key,
method,
headers,
data,
http=self._http)
if resp is None and content is None:
raise KVException("Failed uri={0}, data={1}".format(uri, data))
if resp.status in (200, 201):
return content
elif resp.status == 409:
raise KVAlreadyExists("{0}-{1} already exists".format(uri, data))
elif resp.status == 404:
raise KVNotExists("{0}-{1} not exists".format(uri, data))
else:
raise KVException("Failed to {0} {1}, reason={2}".format(
method, uri, resp.reason))
def content_request(uri, session_key, method, payload, err_msg):
"""
:return: response content if successful otherwise raise
ConfRequestException
"""
resp, content = rest.splunkd_request(uri,
session_key,
method,
data=payload,
retry=3)
if resp is None and content is None:
return None
if resp.status >= 200 and resp.status <= 204:
return content
else:
msg = "{}, status={}, reason={}, detail={}".format(
err_msg, resp.status, resp.reason, content)
logger.error(msg)
if resp.status == 404:
raise ConfNotExistsException(msg)
if resp.status == 409:
raise ConfExistsException(msg)
else:
if content and "already exists" in content:
raise ConfExistsException(msg)
raise ConfRequestException(msg)
def content_request(uri, session_key, method, payload, err_msg):
"""
:return: response content if successful otherwise raise
ConfRequestException
"""
resp, content = rest.splunkd_request(uri, session_key, method,
data=payload, retry=3)
if resp is None and content is None:
return None
if resp.status >= 200 and resp.status <= 204:
return content
else:
msg = "%s, status=%s, reason=%s, detail=%s" % (
err_msg, resp.status, resp.reason, content.decode('utf-8'))
if not (method == "GET" and resp.status == 404):
logger.error(msg)
if resp.status == 404:
raise ConfNotExistsException(msg)
if resp.status == 409:
raise ConfExistsException(msg)
else:
if content and "already exists" in content:
raise ConfExistsException(msg)
raise ConfRequestException(msg)
def content_request(uri, session_key, method, payload, err_msg):
"""
:return: response content if successful otherwise raise
ConfRequestException
"""
resp, content = rest.splunkd_request(uri,
session_key,
method,
data=payload,
retry=3)
if resp is None and content is None:
return None
if resp.status >= 200 and resp.status <= 204:
return content
else:
msg = "{}, status={}, reason={}, detail={}".format(
err_msg,
resp.status,
resp.reason,
content.decode("utf-8"),
)
if not (method == "GET" and resp.status == 404):
log.logger.error(msg)
if resp.status == 404:
raise ConfNotExistsException(msg)
if resp.status == 409:
raise ConfExistsException(msg)
else:
if content and "already exists" in content:
raise ConfExistsException(msg)
raise ConfRequestException(msg)
def get_global_settings(server_uri,
session_key,
user="******",
app=ggc.splunk_ta_google):
"""
:param: get global settings for global settings
:return: a dict of dict which contains global settings .
"""
url = ("{server_uri}/servicesNS/{user}/{app}/splunk_ta_google"
"/google_settings?--get-clear-credential--=1").format(
server_uri=server_uri, user=user, app=app)
response, content = sr.splunkd_request(url, session_key, method="GET")
if not response or response.status not in (200, 201):
raise Exception("Failed to get google global settings."
"Check util log for more details %s" % url)
stanzas = xdp.parse_conf_xml_dom(content)
settings = {}
for stanza in stanzas:
settings[stanza[ggc.name]] = stanza
if not utils.is_true(settings[ggc.proxy_settings].get(ggc.proxy_enabled)):
settings[ggc.proxy_settings][ggc.proxy_url] = None
return settings
def get_google_creds(server_uri,
session_key,
user="******",
app=ggc.splunk_ta_google,
cred_name=""):
"""
:param: get clear creds for cred_name.
:return: a dict of dict which contains all creds. if cred_name is in
(None, ""), return all creds
"""
if not cred_name:
cred_name = ""
url = ("{server_uri}/servicesNS/{user}/{app}/splunk_ta_google"
"/google_credentials/{name}?--get-clear-credential--=1").format(
server_uri=server_uri,
user=user,
app=app,
name=urllib.quote(cred_name, safe=''),
)
response, content = sr.splunkd_request(url, session_key, method="GET")
if not response or response.status not in (200, 201):
raise Exception("Failed to get google credentials for name={}. "
"Check util log for more details".format(cred_name))
stanzas = xdp.parse_conf_xml_dom(content)
creds = {}
for stanza in stanzas:
cred = json.loads(stanza[ggc.google_credentials])
stanza[ggc.google_credentials] = cred
creds[stanza[ggc.name]] = stanza
return creds
def _get_credentials(self, prop, name=None):
"""
:return: clear or encrypted password for specified realm, user
"""
endpoint = self._get_endpoint(name, True)
response, content = rest.splunkd_request(
endpoint,
self._session_key,
method="GET")
if response is None and content is None:
raise CredException("Failed to get clear credentials")
results = {}
if response and response.status in (200, 201) and content:
passwords = xdp.parse_conf_xml_dom(content)
for password in passwords:
if password.get("realm") == self._realm:
values = password[prop].split(self._sep)
if len(values) % 2 == 1:
continue
result = {
values[i]: values[i + 1]
for i in range(0, len(values), 2)
}
results[password.get("username")] = result
return results
def _do_send_safe(self, clusters, dispatched_tasks, forwarder_stanza,
forwarder, settings, localhost):
session_key = self._get_forwarder_session_key(forwarder)
if not localhost and c.global_settings in settings["config_settings"]:
logger.info("Config global settings for forwarder=%s",
forwarder[c.hostname])
uri = ("{}/servicesNS/nobody/{}/kafka_input_setup"
"/kafka_settings/kafka_settings")
url = uri.format(forwarder[c.hostname], self._appname)
payload = {c.all_settings: json.dumps(settings["config_settings"])}
resp, content = rest.splunkd_request(url,
session_key,
method="POST",
data=payload)
if resp is None or resp.status not in (200, 201):
logger.error(
"Failed to update config settings on forwarder=%s",
forwarder[c.hostname])
self._dispatch_errors = True
return
conf_mgr = cm.ConfManager(forwarder[c.hostname],
session_key,
app_name=self._appname)
for name, datainput in settings["data_input_settings"].iteritems():
res = self._create_or_update(clusters, dispatched_tasks,
forwarder_stanza, datainput, name,
forwarder, conf_mgr)
if not res:
self._dispatch_errors = True
def get_session_key(username,
password,
splunkd_uri="https://localhost:8089"):
"""
Get session key by using login username and passwrod
:return: session_key if successful, None if failed
"""
eid = "".join((splunkd_uri, "/services/auth/login"))
postargs = {"username": username, "password": password, }
response, content = rest.splunkd_request(
eid,
None,
method="POST",
data=postargs)
if response is None and content is None:
raise CredException("Get session key failed.")
xml_obj = xdm.parseString(content)
session_nodes = xml_obj.getElementsByTagName("sessionKey")
if not session_nodes:
raise CredException("Invalid username or password.")
session_key = session_nodes[0].firstChild.nodeValue
if not session_key:
raise CredException("Get session key failed.")
return session_key
def _delete(self, name, throw=False):
"""
Delete the encrypted entry
"""
endpoint = self._get_endpoint(name)
response, content = rest.splunkd_request(
endpoint, self._session_key, method="DELETE")
if not response or response.status not in (200, 201, "200", "201"):
if throw:
raise CredException(
"Failed to delete credential stanza {}".format(name))
def delete(self, name):
"""
Delete the encrypted entry
@return: True for success, False for failure
"""
endpoint = self._get_endpoint(name)
response, content = rest.splunkd_request(endpoint,
self._session_key,
method="DELETE")
if response and response.status in (200, 201):
return True
return False
def handleRemove(self, confInfo):
try:
for ep in self._depended_endpoints:
url = self.make_endpoint_url(ep.get('endpoint'))
resp, cont = splunkd_request(url,
self.getSessionKey(),
data={'output_mode': 'json'})
if resp.status not in (200, '200'):
raise Exception(cont)
res = json.loads(cont)
self.check_entries(ep, res['entry'])
except Exception, exc:
RH_Err.ctl(1105, exc)
def delete(self, name, throw=False):
"""
Delete the encrypted entry
"""
endpoint = self._get_endpoint(name)
response, content = rest.splunkd_request(endpoint,
self._session_key,
method="DELETE")
if not response or response not in (200, 201):
if throw:
raise CredException(
"Failed to delete credential stanza {}".format(name))
def _do_rest(uri, session_key):
resp, content = rest.splunkd_request(uri, session_key)
if resp is None:
return None
if resp.status not in (200, 201):
return None
stanza_objs = xdp.parse_conf_xml_dom(content)
if not stanza_objs:
return None
return stanza_objs[0]
def load_config(url, session_key, config_label):
"""
Get AWS configuration.
:param url:
:param session_key:
:param config_label:
:return:
"""
resp, cont = splunkd_request(url, session_key, retry=3)
if resp is None or resp.status not in (200, '200'):
raise Exception('Fail to load %s - %s' % (config_label, cont))
cont = json.loads(cont)
return {ent['name']: ent['content'] for ent in cont['entry']}
def _content_request(uri, session_key, method, payload, err_msg):
resp, content = rest.splunkd_request(uri,
session_key,
method,
data=payload,
retry=3)
if resp is None and content is None:
return None
if resp.status >= 200 and resp.status <= 204:
return content
else:
_LOGGER.error("%s, reason=%s", err_msg, resp.reason)
return None
def _do_update(self, name, password):
try:
self._create(name, password)
except CredException:
payload = {"password": password}
endpoint = self._get_endpoint(name)
response, _ = rest.splunkd_request(endpoint,
self._session_key,
method="POST",
data=payload)
if not response or response.status not in (200, 201):
raise CredException(
"Unable to update password for username={}, status={}".
format(name, response.status))
def _do_request(self, uri, method, payload, err_msg):
resp, content = rest.splunkd_request(uri,
self.session_key,
method,
data=payload,
retry=3)
if resp is None and content is None:
return None
if resp.status in (200, 201):
return content
else:
_LOGGER.debug("%s, reason=%s", err_msg, resp.reason)
return None
def clean(self, endpoint):
url = self.make_uri(endpoint) + "?count=-1"
resp, cont = splunkd_request(
url,
self.getSessionKey(),
method="GET",
data={"output_mode": "json"},
retry=3,
)
if resp is None:
return {url: "Unknown reason"}
if resp.status != 200:
return {url: self.convertErrMsg(cont)}
cont = json.loads(cont)
ents = [ent["name"] for ent in cont.get("entry", [])]
errs = {}
for ent in ents:
args = self.getArgs()
if not self.distinguish(endpoint, ent, **args):
continue
url_ent = self.make_uri(endpoint, entry=ent)
resp, cont = splunkd_request(
url_ent,
self.getSessionKey(),
method="DELETE",
data={"output_mode": "json"},
retry=3,
)
if resp is None:
errs[url_ent] = "Unknown reason"
if resp.status != 200:
errs[url_ent] = self.convertErrMsg(cont)
return errs
def get_default_metric_names(config, namespace):
# Get default metric names for standard namespace
url = ("{server_uri}/servicesNS/-/-/splunk_ta_aws/"
"splunk_ta_aws_cloudwatch_default_settings?namespace={namespace}&"
"output_mode=json").format(server_uri=config["server_uri"],
namespace=namespace)
response, content = sr.splunkd_request(url, config[tac.session_key])
if not response or response.status not in (200, 201):
return None
content = json.loads(content)
metrics = json.loads(content["entry"][0]["content"]["metrics"])
logger.info("Got default metric_names=%s for namespace=%s", metrics,
namespace)
return metrics
def delete(self, name):
"""
Delete the encrypted entry
@return: True for success, False for failure
"""
allpwd = self.get_all_passwords()
if name in [pw.get("username") for pw in allpwd]:
endpoint = self._get_endpoint(name)
response, content = rest.splunkd_request(endpoint,
self._session_key,
method="DELETE")
if response and response.status in (200, 201):
return True
return False
def getProxyInfo(self, splunkdMgmtUri, sessionKey, user, app):
"""
Get Proxy Information on given REST endpoint.
It should be in form (if not, override the this method):
{
'proxy_url': '<proxy_url>',
'proxy_port': '<proxy_port>',
'proxy_username': '******',
'proxy_password': '******',
'proxy_enabled': '<proxy_enabled>',
'proxy_rdns': '<proxy_rdns>',
'proxy_type': '<proxy_type>'
}
:return:
"""
if not self.proxyInfoEndpoint:
RH_Err.ctl(1104, msgx="Empty endpoint for proxy is being used")
url = "{splunkdMgmtUri}servicesNS/{user}/{app}/{proxyInfoEndpoint}".format(
splunkdMgmtUri=splunkdMgmtUri,
user=user,
app=app,
proxyInfoEndpoint=self.proxyInfoEndpoint,
)
data = {"output_mode": "json", "--get-clear-credential--": "1"}
resp, cont = splunkd_request(url, sessionKey, data=data, retry=3)
if resp is None or resp.status != 200:
RH_Err.ctl(
1104,
msgx="failed to load proxy info. {err}".format(
err=code_to_msg(resp, cont) if resp else cont
),
)
try:
proxy_info = json.loads(cont)["entry"][0]["content"]
except IndexError | KeyError:
proxy_info = {}
return {
"proxy_enabled": is_true(proxy_info.get("proxy_enabled", "false")),
"proxy_url": proxy_info.get("proxy_url", ""),
"proxy_port": proxy_info.get("proxy_port", ""),
"proxy_username": proxy_info.get("proxy_username", ""),
"proxy_password": proxy_info.get("proxy_password", ""),
"proxy_rdns": proxy_info.get("proxy_rdns", "false"),
"proxy_type": proxy_info.get("proxy_type", "http"),
}
def load(self):
"""Load Configurations in UCC according to the schema
It will raise exception if failing to load any endpoint,
because it make no sense with not complete configuration info.
"""
log('"load" method in', level=logging.DEBUG)
ret = {
meta_field: getattr(self, meta_field)
for meta_field in Config.META_FIELDS
}
for ep_id, ep in self._endpoints.items():
data = {"output_mode": "json", "--cred--": "1"}
retries = 4
waiting_time = [1, 2, 2]
for retry in range(retries):
resp, cont = splunkd_request(
splunkd_uri=self.make_uri(ep_id),
session_key=self.session_key,
data=data,
retry=3,
)
if resp is None or resp.status != 200:
msg = 'Fail to load endpoint "{ep_id}" - {err}' "".format(
ep_id=ep_id,
err=code_to_msg(resp, cont) if resp else cont)
log(msg, level=logging.ERROR, need_tb=True)
raise ConfigException(msg)
try:
ret[ep_id] = self._parse_content(ep_id, cont)
except ConfigException as exc:
log(exc, level=logging.WARNING, need_tb=True)
if retry < retries - 1:
time.sleep(waiting_time[retry])
else:
break
else:
log(exc, level=logging.ERROR, need_tb=True)
raise exc
log('"load" method out', level=logging.DEBUG)
return ret
def _create(self, name, str_to_encrypt):
"""
Create a new stored credential.
:return: raise on failure
"""
payload = {
"name": name,
"password": str_to_encrypt,
"realm": self._realm,
}
endpoint = self._get_endpoint(name)
resp, content = rest.splunkd_request(endpoint, self._session_key,
method="POST", data=payload)
if not resp or resp.status not in (200, 201, "200", "201"):
raise CredException("Failed to encrypt username {}".format(name))
def _get_all_passwords(self):
"""
:return: a list of dict when successful, None when failed.
the dict at least contains
{
"realm": xxx,
"username": yyy,
"clear_password": zzz,
}
"""
endpoint = self._get_endpoint()
response, content = rest.splunkd_request(
endpoint, self._session_key, method="GET")
if response and response.status in (200, 201, "200", "201") and content:
return xdp.parse_conf_xml_dom(content)
raise CredException("Failed to get credentials")
def get_all_passwords(self):
"""
@return: a list of dict when successful, None when failed.
the dict at least contains
{
"realm": xxx,
"username": yyy,
"clear_password": zzz,
}
"""
endpoint = "{}/services/storage/passwords".format(self._splunkd_uri)
response, content = rest.splunkd_request(endpoint,
self._session_key,
method="GET")
if response and response.status in (200, 201) and content:
return xdp.parse_conf_xml_dom(content)
def _do_request(self, uri, method, data=None,
content_type="application/x-www-form-urlencoded"):
headers = {"Content-Type": content_type}
resp, content = rest.splunkd_request(uri, self._session_key,
method, headers, data)
if resp is None and content is None:
raise KVException("Failed uri={0}, data={1}".format(uri, data))
if resp.status in (200, 201):
return content
elif resp.status == 409:
raise KVAlreadyExists("{0}-{1} already exists".format(uri, data))
elif resp.status == 404:
raise KVNotExists("{0}-{1} not exists".format(uri, data))
else:
raise KVException("Failed to {0} {1}, reason={2}".format(
method, uri, resp.reason))
def _get_all_passwords(self):
"""
:return: a list of dict when successful, None when failed.
the dict at least contains
{
"realm": xxx,
"username": yyy,
"clear_password": zzz,
}
"""
endpoint = self._get_endpoint()
response, content = rest.splunkd_request(
endpoint, self._session_key, method="GET"
)
if response and response.status in (200, 201, "200", "201") and content:
return xdp.parse_conf_xml_dom(content)
raise CredException("Failed to get credentials")
def _create(self, name, str_to_encrypt):
"""
Create a new stored credential.
:return: raise on failure
"""
payload = {
"name": name,
"password": str_to_encrypt,
"realm": self._realm,
}
endpoint = self._get_endpoint(name)
resp, content = rest.splunkd_request(
endpoint, self._session_key, method="POST", data=payload
)
if not resp or resp.status not in (200, 201, "200", "201"):
raise CredException("Failed to encrypt username {}".format(name))
def _delete(self, name, throw=False):
"""
Delete the encrypted entry
"""
endpoint = self._get_endpoint(name)
response, content = rest.splunkd_request(endpoint,
self._session_key,
method="DELETE")
if response is not None and response.status in (404, "404"):
if throw:
raise CredNotFound(
"Credential stanza not exits - {}".format(name))
elif not response or response.status not in (200, 201, "200", "201"):
if throw:
raise CredException(
"Failed to delete credential stanza {}".format(name))
def getProxyInfo(self, splunkdMgmtUri, sessionKey, user, app):
"""
Get Proxy Information on given REST endpoint.
It should be in form (if not, override the this method):
{
'proxy_url': '<proxy_url>',
'proxy_port': '<proxy_port>',
'proxy_username': '******',
'proxy_password': '******',
'proxy_enabled': '<proxy_enabled>',
'proxy_rdns': '<proxy_rdns>',
'proxy_type': '<proxy_type>'
}
:return:
"""
if not self.proxyInfoEndpoint:
RH_Err.ctl(1104, msgx='Empty endpoint for proxy is being used')
url = "{splunkdMgmtUri}servicesNS/{user}/{app}/{proxyInfoEndpoint}"\
.format(splunkdMgmtUri=splunkdMgmtUri,
user=user,
app=app,
proxyInfoEndpoint=self.proxyInfoEndpoint)
data = {'output_mode': 'json', '--get-clear-credential--': '1'}
resp, cont = splunkd_request(url, sessionKey, data=data, retry=3)
if resp is None or resp.status != 200:
RH_Err.ctl(1104,
msgx='failed to load proxy info. {err}'.format(
err=code_to_msg(resp, cont) if resp else cont))
try:
proxy_info = json.loads(cont)['entry'][0]['content']
except IndexError | KeyError:
proxy_info = {}
return {
'proxy_enabled': is_true(proxy_info.get('proxy_enabled', 'false')),
'proxy_url': proxy_info.get('proxy_url', ''),
'proxy_port': proxy_info.get('proxy_port', ''),
'proxy_username': proxy_info.get('proxy_username', ''),
'proxy_password': proxy_info.get('proxy_password', ''),
'proxy_rdns': proxy_info.get('proxy_rdns', 'false'),
'proxy_type': proxy_info.get('proxy_type', 'http')
}
def reload_confs(confs,
session_key,
splunkd_uri="https://localhost:8089",
appname="-"):
new_confs = []
for conf in confs:
conf = op.basename(conf)
if conf.endswith(".conf"):
conf = conf[:-5]
new_confs.append(conf)
else:
new_confs.append(conf)
endpoint_template = "{0}/servicesNS/-/{1}/configs/conf-{2}/_reload"
for conf in new_confs:
endpoint = endpoint_template.format(splunkd_uri, appname, conf)
resp, _ = rest.splunkd_request(endpoint, session_key)
if not resp or resp.status not in (200, 201):
_LOGGER.error("Failed to refresh %s, reason=%s", endpoint,
resp.reason if resp else "")
def user_caps(mgmt_uri, session_key):
"""
Get capabilities of sessioned Splunk user.
:param mgmt_uri:
:param session_key:
:return:
"""
url = mgmt_uri + "/services/authentication/current-context"
resp = splunkd_request(
url, session_key, method="GET", data={"output_mode": "json"}, retry=3
)
if resp is None:
RH_Err.ctl(500, logging.ERROR, "Fail to get capabilities of sessioned user")
elif resp.status_code != 200:
RH_Err.ctl(resp.status_code, logging.ERROR, resp.text)
cont = resp.json()
caps = cont["entry"][0]["content"]["capabilities"]
return set(caps)