def process_params(self, orig_params): # pylint: disable=no-self-use
"""Process parameters into usable pieces.
Override this if you provide any config parameters that may require
interpreation, such as the relatime parameter
"""
params = orig_params.copy()
for (key, value) in params.items():
if hasattr(value, "items"):
conf = params.pop(key)
if "relatime" in conf:
local_tz = get_localzone()
dto = relatime.timeParser(conf["relatime"],
timezone=str(local_tz))
target_tz = pytz.timezone(conf.get("timezone", "UTC"))
dto = dto.astimezone(target_tz)
dto = dto.replace(tzinfo=None)
time_format = conf.get("format", "%Y-%m-%dT%H:%M:%S.%fZ")
if time_format.lower() == "as_epoch":
try:
timestamp = dto.timestamp()
except AttributeError:
import calendar
timestamp = calendar.timegm(dto.timetuple())
params[key] = str(int(timestamp))
else:
params[key] = dto.strftime(time_format)
return params
def _req(self, conf, url=None):
if url is None:
url = conf.get("url", "")
if url == "":
return
url = url.format(**self.kwargs)
method = conf.get("method", "get").upper()
kwargs = dict()
headers = conf.get("headers", {})
if headers:
kwargs["headers"] = headers
verify_ssl = conf.get("verify_ssl", True)
# GET params
params = conf.get("params", {}).copy()
for (k, v) in params.items():
if hasattr(v, "items"):
conf = params.pop(k)
if "relatime" in conf:
dt = relatime.timeParser(conf["relatime"],
timezone=str(get_localzone()))
target_tz = pytz.timezone(conf.get("timezone", "UTC"))
dt = dt.astimezone(target_tz)
dt = dt.replace(tzinfo=None)
time_format = conf.get("format", "%Y-%m-%dT%H:%M:%S.%fZ")
if time_format.lower() == "as_epoch":
params[k] = str(int(dt.timestamp()))
else:
params[k] = dt.strftime(time_format)
else:
params[k] = str(v).format(**self.kwargs)
if params:
kwargs["params"] = params
# POST data
data = conf.get("data", {})
for (k, v) in data.items():
data[k] = v.format(**self.kwargs)
if data:
kwargs["data"] = data
# HTTP Basic Auth
if conf.get("auth") and self.creds and self.creds.get(conf["auth"]):
kwargs["auth"] = tuple(self.creds[conf["auth"]])
# Auto decompress
if conf.get("decompress", False):
kwargs["hooks"] = {"response": self.unzip_content}
raw_req = requests.Request(method, url, **kwargs)
req = self.session.prepare_request(raw_req)
if self.kwargs.get("verbose", False):
print("[.] Requesting {0} ({1})".format(req.url, req.method))
with warnings.catch_warnings():
if not verify_ssl:
warnings.simplefilter("ignore",
exceptions.InsecureRequestWarning)
return self.session.send(req, verify=verify_ssl)
def _req(self, conf, url=None):
if url is None:
url = conf.get("url", "")
if url == "":
return
url = url.format(**self.kwargs)
method = conf.get("method", "get").upper()
kwargs = dict()
headers = conf.get("headers", {})
if len(headers) > 0:
kwargs["headers"] = headers
verify_ssl = conf.get("verify_ssl", True)
# GET params
params = conf.get("params", {}).copy()
for (k, v) in params.items():
if hasattr(v, "items"):
conf = params.pop(k)
if "relatime" in conf:
dt = relatime.timeParser(conf["relatime"], timezone=str(get_localzone()))
target_tz = pytz.timezone(conf.get("timezone", "UTC"))
dt = dt.astimezone(target_tz)
dt = dt.replace(tzinfo=None)
time_format = conf.get("format", "%Y-%m-%dT%H:%M:%S.%fZ")
if time_format.lower() == "as_epoch":
params[k] = str(int(dt.timestamp()))
else:
params[k] = dt.strftime(time_format)
else:
params[k] = str(v).format(**self.kwargs)
if len(params) > 0:
kwargs["params"] = params
# POST data
data = conf.get("data", {})
for (k, v) in data.items():
data[k] = v.format(**self.kwargs)
if len(data) > 0:
kwargs["data"] = data
# HTTP Basic Auth
if conf.get("auth") and self.creds and self.creds.get(conf["auth"]):
kwargs["auth"] = tuple(self.creds[conf["auth"]])
# Auto decompress
if conf.get("decompress", False):
kwargs["hooks"] = {"response": self.unzip_content}
raw_req = requests.Request(method, url, **kwargs)
req = self.session.prepare_request(raw_req)
if self.kwargs.get("verbose", False):
print("[.] Requesting {0} ({1})".format(req.url, req.method))
with warnings.catch_warnings():
if not verify_ssl:
warnings.simplefilter("ignore", exceptions.InsecureRequestWarning)
return self.session.send(req, verify=verify_ssl)
def test_get_params_relatime_timezone(self):
import relatime
import pytz
from tzlocal import get_localzone
param_key = fauxfactory.gen_alpha().lower()
param_value = {"relatime": "-1d@d", "timezone": "America/New_York"}
expected = relatime.timeParser("-1d@d", timezone=str(get_localzone()))
target_tz = pytz.timezone("America/New_York")
expected = expected.astimezone(target_tz)
expected = expected.replace(tzinfo=None)
r = self.do_request("http://httpbin.org/get",
params={param_key: param_value})
data = r.json().get("args", {})
self.assertEqual(data.get(param_key, None),
expected.strftime("%Y-%m-%dT%H:%M:%S.%fZ"))
def test_get_params_relatime_epoch(self):
import relatime
from tzlocal import get_localzone
tz = get_localzone()
param_key = fauxfactory.gen_alpha().lower()
param_value = {
"relatime": "-1d@d",
"format": "as_epoch",
"timezone": str(tz)
}
expected = relatime.timeParser("-1d@d", timezone=str(tz))
r = self.do_request("http://httpbin.org/get",
params={param_key: param_value})
data = r.json().get("args", {})
try:
expected = expected.timestamp()
except AttributeError:
import calendar
expected = calendar.timegm(expected.timetuple())
self.assertEqual(data.get(param_key, None), str(int(expected)))
def multi_match_generator(cls, data, parser, mm_key):
if not hasattr(data, "items"):
# Is a list, process each list item
for item in data:
for _ in cls.multi_match_generator(item, parser, mm_key="@"):
yield _
return
onlyif = parser.get("onlyif", None)
if onlyif is not None and not hasattr(onlyif, "items"):
onlyif = {"key": onlyif}
# Decide how to iterate on the data
# Options are:
# Return result_dict per match in dict (if: data is dict)
# Return one result_dict for whole dict (if: data is dict)
if mm_key == "@" or parser.get("match_all", False):
# Treat the entire data as a single match
# Returns a single result_dict
data = [(None, data)]
else:
# Each matching key is a separate result_dict
data = data.items()
for (k, v) in data:
if onlyif is not None:
if not hasattr(onlyif, "items"):
onlyif = {"key": onlyif}
value = cls.get_value(v, onlyif["key"], None)
if value is None:
continue
elif "regex" in onlyif:
rex = re.compile(onlyif["regex"], re.I)
if not rex.search(value):
continue
# Check for maxage key in onlyif. If it exists, parse it as Splunk relative time syntax and compare to parsed input "value"
elif "maxage" in onlyif:
age = parse(value)
if not onlyif["maxage"].startswith(
"-"): # Assume we want dates in the past
print(
'\033[91m' +
'WARNING: maxage must be prepended with "-" Please correct this in your configuration file.'
+ '\033[0m')
onlyif["maxage"] = "-%s" % onlyif["maxage"]
ageout = timeParser(onlyif["maxage"]).replace(tzinfo=None)
if age < ageout:
continue
else:
if not bool(value):
continue
result_dict = OrderedDict()
for mm_parser in parser["keys"]:
for mm_result_dict in cls.get_result_dicts(v,
mm_parser,
mm_key=k,
onlyif=onlyif):
result_dict.update(mm_result_dict)
if result_dict:
result_dict.labels = parser.get("labels", None)
yield result_dict