class LRUDict(object):
def __init__(self, capacity):
self.capacity = capacity
self.cache = OrderedDict()
def __len__(self):
return len(self.cache)
def __contains__(self, key):
return key in self.cache
def __getitem__(self, key):
value = self.cache.pop(key)
self.cache[key] = value
return value
def get(self, key):
return self.__getitem__(key)
def __setitem__(self, key, value):
try:
self.cache.pop(key)
except KeyError:
if len(self.cache) >= self.capacity:
self.cache.popitem(last=False)
self.cache[key] = value
def set(self, key, value):
self.__setitem__(key, value)
def keys(self):
return self.cache.keys()
class LRUDict(object):
"""
This class defines the LRU dictionary
>>> foo = LRUDict(capacity=2)
>>> foo["first"] = 1
>>> foo["second"] = 2
>>> foo["third"] = 3
>>> "first" in foo
False
>>> "third" in foo
True
"""
def __init__(self, capacity):
self.capacity = capacity
self.cache = OrderedDict()
def __len__(self):
return len(self.cache)
def __contains__(self, key):
return key in self.cache
def __getitem__(self, key):
try:
value = self.cache.pop(key)
self.cache[key] = value
except:
value = None
return value
def get(self, key):
return self.__getitem__(key)
def __setitem__(self, key, value):
try:
self.cache.pop(key)
except KeyError:
if len(self.cache) >= self.capacity:
self.cache.popitem(last=False)
self.cache[key] = value
def set(self, key, value):
self.__setitem__(key, value)
def keys(self):
return self.cache.keys()
def _setRequestParams():
"""
Check and set the parameters and perform checks on 'data' option for
HTTP method POST.
"""
if conf.direct:
conf.parameters[None] = "direct connection"
return
hintNames = []
testableParameters = False
# Perform checks on GET parameters
if conf.parameters.get(PLACE.GET):
parameters = conf.parameters[PLACE.GET]
paramDict = paramToDict(PLACE.GET, parameters)
if paramDict:
conf.paramDict[PLACE.GET] = paramDict
testableParameters = True
# Perform checks on POST parameters
if conf.method == HTTPMETHOD.POST and conf.data is None:
logger.warn("detected empty POST body")
conf.data = ""
if conf.data is not None:
conf.method = HTTPMETHOD.POST if not conf.method or conf.method == HTTPMETHOD.GET else conf.method
def process(match, repl):
retVal = match.group(0)
if not (conf.testParameter
and match.group("name") not in conf.testParameter):
retVal = repl
while True:
_ = re.search(r"\\g<([^>]+)>", retVal)
if _:
retVal = retVal.replace(
_.group(0),
match.group(
int(_.group(1)) if _.group(1).isdigit() else _.
group(1)))
else:
break
if kb.customInjectionMark in retVal:
hintNames.append((retVal.split(kb.customInjectionMark)[0],
match.group("name")))
return retVal
if kb.processUserMarks is None and kb.customInjectionMark in conf.data:
message = "custom injection marker ('%s') found in option " % kb.customInjectionMark
message += "'--data'. Do you want to process it? [Y/n/q] "
choice = readInput(message, default='Y').upper()
if choice == 'Q':
raise SqlmapUserQuitException
else:
kb.processUserMarks = choice == 'Y'
if kb.processUserMarks:
kb.testOnlyCustom = True
if re.search(JSON_RECOGNITION_REGEX, conf.data):
message = "JSON data found in %s data. " % conf.method
message += "Do you want to process it? [Y/n/q] "
choice = readInput(message, default='Y').upper()
if choice == 'Q':
raise SqlmapUserQuitException
elif choice == 'Y':
if not (kb.processUserMarks
and kb.customInjectionMark in conf.data):
conf.data = getattr(conf.data, UNENCODED_ORIGINAL_VALUE,
conf.data)
conf.data = conf.data.replace(kb.customInjectionMark,
ASTERISK_MARKER)
conf.data = re.sub(
r'("(?P<name>[^"]+)"\s*:\s*".+?)"(?<!\\")',
functools.partial(process,
repl=r'\g<1>%s"' %
kb.customInjectionMark), conf.data)
conf.data = re.sub(
r'("(?P<name>[^"]+)"\s*:\s*)(-?\d[\d\.]*)\b',
functools.partial(process,
repl=r'\g<1>\g<3>%s' %
kb.customInjectionMark), conf.data)
conf.data = re.sub(
r'("(?P<name>[^"]+)"\s*:\s*)((true|false|null))\b',
functools.partial(process,
repl=r'\g<1>\g<3>%s' %
kb.customInjectionMark), conf.data)
match = re.search(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]',
conf.data)
if match and not (conf.testParameter
and match.group("name")
not in conf.testParameter):
_ = match.group(2)
_ = re.sub(r'("[^"]+)"',
r'\g<1>%s"' % kb.customInjectionMark, _)
_ = re.sub(r'(\A|,|\s+)(-?\d[\d\.]*\b)',
r'\g<0>%s' % kb.customInjectionMark, _)
conf.data = conf.data.replace(
match.group(0),
match.group(0).replace(match.group(2), _))
kb.postHint = POST_HINT.JSON
elif re.search(JSON_LIKE_RECOGNITION_REGEX, conf.data):
message = "JSON-like data found in %s data. " % conf.method
message += "Do you want to process it? [Y/n/q] "
choice = readInput(message, default='Y').upper()
if choice == 'Q':
raise SqlmapUserQuitException
elif choice == 'Y':
if not (kb.processUserMarks
and kb.customInjectionMark in conf.data):
conf.data = getattr(conf.data, UNENCODED_ORIGINAL_VALUE,
conf.data)
conf.data = conf.data.replace(kb.customInjectionMark,
ASTERISK_MARKER)
conf.data = re.sub(
r"('(?P<name>[^']+)'\s*:\s*'[^']+)'",
functools.partial(process,
repl=r"\g<1>%s'" %
kb.customInjectionMark), conf.data)
conf.data = re.sub(
r"('(?P<name>[^']+)'\s*:\s*)(-?\d[\d\.]*\b)",
functools.partial(process,
repl=r"\g<0>%s" %
kb.customInjectionMark), conf.data)
kb.postHint = POST_HINT.JSON_LIKE
elif re.search(ARRAY_LIKE_RECOGNITION_REGEX, conf.data):
message = "Array-like data found in %s data. " % conf.method
message += "Do you want to process it? [Y/n/q] "
choice = readInput(message, default='Y').upper()
if choice == 'Q':
raise SqlmapUserQuitException
elif choice == 'Y':
if not (kb.processUserMarks
and kb.customInjectionMark in conf.data):
conf.data = conf.data.replace(kb.customInjectionMark,
ASTERISK_MARKER)
conf.data = re.sub(
r"(=[^%s]+)" % DEFAULT_GET_POST_DELIMITER,
r"\g<1>%s" % kb.customInjectionMark, conf.data)
kb.postHint = POST_HINT.ARRAY_LIKE
elif re.search(XML_RECOGNITION_REGEX, conf.data):
message = "SOAP/XML data found in %s data. " % conf.method
message += "Do you want to process it? [Y/n/q] "
choice = readInput(message, default='Y').upper()
if choice == 'Q':
raise SqlmapUserQuitException
elif choice == 'Y':
if not (kb.processUserMarks
and kb.customInjectionMark in conf.data):
conf.data = getattr(conf.data, UNENCODED_ORIGINAL_VALUE,
conf.data)
conf.data = conf.data.replace(kb.customInjectionMark,
ASTERISK_MARKER)
conf.data = re.sub(
r"(<(?P<name>[^>]+)( [^<]*)?>)([^<]+)(</\2)",
functools.partial(process,
repl=r"\g<1>\g<4>%s\g<5>" %
kb.customInjectionMark), conf.data)
kb.postHint = POST_HINT.SOAP if "soap" in conf.data.lower(
) else POST_HINT.XML
elif re.search(MULTIPART_RECOGNITION_REGEX, conf.data):
message = "Multipart-like data found in %s data. " % conf.method
message += "Do you want to process it? [Y/n/q] "
choice = readInput(message, default='Y').upper()
if choice == 'Q':
raise SqlmapUserQuitException
elif choice == 'Y':
if not (kb.processUserMarks
and kb.customInjectionMark in conf.data):
conf.data = getattr(conf.data, UNENCODED_ORIGINAL_VALUE,
conf.data)
conf.data = conf.data.replace(kb.customInjectionMark,
ASTERISK_MARKER)
conf.data = re.sub(
r"(?si)((Content-Disposition[^\n]+?name\s*=\s*[\"']?(?P<name>[^\"'\r\n]+)[\"']?).+?)(((\r)?\n)+--)",
functools.partial(process,
repl=r"\g<1>%s\g<4>" %
kb.customInjectionMark), conf.data)
kb.postHint = POST_HINT.MULTIPART
if not kb.postHint:
if kb.customInjectionMark in conf.data: # later processed
pass
else:
place = PLACE.POST
conf.parameters[place] = conf.data
paramDict = paramToDict(place, conf.data)
if paramDict:
conf.paramDict[place] = paramDict
testableParameters = True
else:
if kb.customInjectionMark not in conf.data: # in case that no usable parameter values has been found
conf.parameters[PLACE.POST] = conf.data
kb.processUserMarks = True if (kb.postHint and kb.customInjectionMark in (
conf.data or "")) else kb.processUserMarks
if re.search(URI_INJECTABLE_REGEX, conf.url, re.I) and not any(
place in conf.parameters for place in
(PLACE.GET,
PLACE.POST)) and not kb.postHint and kb.customInjectionMark not in (
conf.data or "") and conf.url.startswith("http"):
warnMsg = "you've provided target URL without any GET "
warnMsg += "parameters (e.g. 'http://www.site.com/article.php?id=1') "
warnMsg += "and without providing any POST parameters "
warnMsg += "through option '--data'"
logger.warn(warnMsg)
message = "do you want to try URI injections "
message += "in the target URL itself? [Y/n/q] "
choice = readInput(message, default='Y').upper()
if choice == 'Q':
raise SqlmapUserQuitException
elif choice == 'Y':
conf.url = "%s%s" % (conf.url, kb.customInjectionMark)
kb.processUserMarks = True
for place, value in ((PLACE.URI, conf.url), (PLACE.CUSTOM_POST, conf.data),
(PLACE.CUSTOM_HEADER, str(conf.httpHeaders))):
_ = re.sub(PROBLEMATIC_CUSTOM_INJECTION_PATTERNS, "", value
or "") if place == PLACE.CUSTOM_HEADER else value or ""
if kb.customInjectionMark in _:
if kb.processUserMarks is None:
lut = {
PLACE.URI:
'-u',
PLACE.CUSTOM_POST:
'--data',
PLACE.CUSTOM_HEADER:
'--headers/--user-agent/--referer/--cookie'
}
message = "custom injection marker ('%s') found in option " % kb.customInjectionMark
message += "'%s'. Do you want to process it? [Y/n/q] " % lut[
place]
choice = readInput(message, default='Y').upper()
if choice == 'Q':
raise SqlmapUserQuitException
else:
kb.processUserMarks = choice == 'Y'
if kb.processUserMarks:
kb.testOnlyCustom = True
if "=%s" % kb.customInjectionMark in _:
warnMsg = "it seems that you've provided empty parameter value(s) "
warnMsg += "for testing. Please, always use only valid parameter values "
warnMsg += "so sqlmap could be able to run properly"
logger.warn(warnMsg)
if not kb.processUserMarks:
if place == PLACE.URI:
query = urlparse.urlsplit(value).query
if query:
parameters = conf.parameters[PLACE.GET] = query
paramDict = paramToDict(PLACE.GET, parameters)
if paramDict:
conf.url = conf.url.split('?')[0]
conf.paramDict[PLACE.GET] = paramDict
testableParameters = True
elif place == PLACE.CUSTOM_POST:
conf.parameters[PLACE.POST] = conf.data
paramDict = paramToDict(PLACE.POST, conf.data)
if paramDict:
conf.paramDict[PLACE.POST] = paramDict
testableParameters = True
else:
conf.parameters[place] = value
conf.paramDict[place] = OrderedDict()
if place == PLACE.CUSTOM_HEADER:
for index in xrange(len(conf.httpHeaders)):
header, value = conf.httpHeaders[index]
if kb.customInjectionMark in re.sub(
PROBLEMATIC_CUSTOM_INJECTION_PATTERNS, "",
value):
parts = value.split(kb.customInjectionMark)
for i in xrange(len(parts) - 1):
conf.paramDict[place]["%s #%d%s" % (
header, i + 1, kb.customInjectionMark
)] = "%s,%s" % (header, "".join(
"%s%s" %
(parts[j],
kb.customInjectionMark if i == j else "")
for j in xrange(len(parts))))
conf.httpHeaders[index] = (
header,
value.replace(kb.customInjectionMark, ""))
else:
parts = value.split(kb.customInjectionMark)
for i in xrange(len(parts) - 1):
name = None
if kb.postHint:
for ending, _ in hintNames:
if parts[i].endswith(ending):
name = "%s %s" % (kb.postHint, _)
break
if name is None:
name = "%s#%s%s" % (
("%s " % kb.postHint) if kb.postHint else "",
i + 1, kb.customInjectionMark)
conf.paramDict[place][name] = "".join(
"%s%s" % (parts[j],
kb.customInjectionMark if i == j else "")
for j in xrange(len(parts)))
if place == PLACE.URI and PLACE.GET in conf.paramDict:
del conf.paramDict[PLACE.GET]
elif place == PLACE.CUSTOM_POST and PLACE.POST in conf.paramDict:
del conf.paramDict[PLACE.POST]
testableParameters = True
if kb.processUserMarks:
for item in ("url", "data", "agent", "referer", "cookie"):
if conf.get(item):
conf[item] = conf[item].replace(kb.customInjectionMark, "")
# Perform checks on Cookie parameters
if conf.cookie:
conf.parameters[PLACE.COOKIE] = conf.cookie
paramDict = paramToDict(PLACE.COOKIE, conf.cookie)
if paramDict:
conf.paramDict[PLACE.COOKIE] = paramDict
testableParameters = True
# Perform checks on header values
if conf.httpHeaders:
for httpHeader, headerValue in list(conf.httpHeaders):
# Url encoding of the header values should be avoided
# Reference: http://stackoverflow.com/questions/5085904/is-ok-to-urlencode-the-value-in-headerlocation-value
if httpHeader.upper() == HTTP_HEADER.USER_AGENT.upper():
conf.parameters[PLACE.USER_AGENT] = urldecode(headerValue)
condition = any((not conf.testParameter,
intersect(conf.testParameter,
USER_AGENT_ALIASES, True)))
if condition:
conf.paramDict[PLACE.USER_AGENT] = {
PLACE.USER_AGENT: headerValue
}
testableParameters = True
elif httpHeader.upper() == HTTP_HEADER.REFERER.upper():
conf.parameters[PLACE.REFERER] = urldecode(headerValue)
condition = any((not conf.testParameter,
intersect(conf.testParameter, REFERER_ALIASES,
True)))
if condition:
conf.paramDict[PLACE.REFERER] = {
PLACE.REFERER: headerValue
}
testableParameters = True
elif httpHeader.upper() == HTTP_HEADER.HOST.upper():
conf.parameters[PLACE.HOST] = urldecode(headerValue)
condition = any((not conf.testParameter,
intersect(conf.testParameter, HOST_ALIASES,
True)))
if condition:
conf.paramDict[PLACE.HOST] = {PLACE.HOST: headerValue}
testableParameters = True
else:
condition = intersect(conf.testParameter, [httpHeader], True)
if condition:
conf.parameters[PLACE.CUSTOM_HEADER] = str(
conf.httpHeaders)
conf.paramDict[PLACE.CUSTOM_HEADER] = {
httpHeader:
"%s,%s%s" %
(httpHeader, headerValue, kb.customInjectionMark)
}
conf.httpHeaders = [
(_[0], _[1].replace(kb.customInjectionMark, ""))
for _ in conf.httpHeaders
]
testableParameters = True
if not conf.parameters:
errMsg = "you did not provide any GET, POST and Cookie "
errMsg += "parameter, neither an User-Agent, Referer or Host header value"
raise SqlmapGenericException(errMsg)
elif not testableParameters:
errMsg = "all testable parameters you provided are not present "
errMsg += "within the given request data"
raise SqlmapGenericException(errMsg)
if conf.csrfToken:
if not any(
re.search(conf.csrfToken, ' '.join(_), re.I) for _ in
(conf.paramDict.get(PLACE.GET, {}),
conf.paramDict.get(PLACE.POST, {}))) and not re.search(
r"\b%s\b" % conf.csrfToken, conf.data
or "") and conf.csrfToken not in set(
_[0].lower() for _ in conf.httpHeaders
) and conf.csrfToken not in conf.paramDict.get(
PLACE.COOKIE, {}):
errMsg = "anti-CSRF token parameter '%s' not " % conf.csrfToken._original
errMsg += "found in provided GET, POST, Cookie or header values"
raise SqlmapGenericException(errMsg)
else:
for place in (PLACE.GET, PLACE.POST, PLACE.COOKIE):
if conf.csrfToken:
break
for parameter in conf.paramDict.get(place, {}):
if any(parameter.lower().count(_)
for _ in CSRF_TOKEN_PARAMETER_INFIXES):
message = "%s parameter '%s' appears to hold anti-CSRF token. " % (
place, parameter)
message += "Do you want sqlmap to automatically update it in further requests? [y/N] "
if readInput(message, default='N', boolean=True):
class _(unicode):
pass
conf.csrfToken = _(re.escape(getUnicode(parameter)))
conf.csrfToken._original = getUnicode(parameter)
break
def __init__(self, capacity):
self.capacity = capacity
self.cache = OrderedDict()
def unionThread():
threadData = getCurrentThreadData()
while kb.threadContinue:
with kb.locks.limit:
try:
threadData.shared.counter += 1
num = next(threadData.shared.limits)
except StopIteration:
break
if Backend.getIdentifiedDbms() in (DBMS.MSSQL,
DBMS.SYBASE):
field = expressionFieldsList[0]
elif Backend.isDbms(DBMS.ORACLE):
field = expressionFieldsList
else:
field = None
limitedExpr = agent.limitQuery(
num, expression, field)
output = _oneShotUnionUse(limitedExpr, unpack,
True)
if not kb.threadContinue:
break
if output:
with kb.locks.value:
if all(_ in output
for _ in (kb.chars.start,
kb.chars.stop)):
items = parseUnionPage(output)
if threadData.shared.showEta:
threadData.shared.progress.progress(
threadData.shared.counter)
if isListLike(items):
# in case that we requested N columns and we get M!=N then we have to filter a bit
if len(items) > 1 and len(
expressionFieldsList) > 1:
items = [
item for item in items
if isListLike(item)
and len(item) == len(
expressionFieldsList)
]
items = [
_ for _ in flattenValue(items)
]
if len(items) > len(
expressionFieldsList):
filtered = OrderedDict()
for item in items:
key = re.sub(
r"[^A-Za-z0-9]", "",
item).lower()
if key not in filtered or re.search(
r"[^A-Za-z0-9]",
item):
filtered[key] = item
items = list(
six.itervalues(filtered))
items = [items]
index = None
for index in xrange(1 + len(
threadData.shared.buffered)):
if index < len(
threadData.shared.buffered
) and threadData.shared.buffered[
index][0] >= num:
break
threadData.shared.buffered.insert(
index or 0, (num, items))
else:
index = None
if threadData.shared.showEta:
threadData.shared.progress.progress(
threadData.shared.counter)
for index in xrange(1 + len(
threadData.shared.buffered)):
if index < len(
threadData.shared.buffered
) and threadData.shared.buffered[
index][0] >= num:
break
threadData.shared.buffered.insert(
index or 0, (num, None))
items = output.replace(
kb.chars.start, "").replace(
kb.chars.stop,
"").split(kb.chars.delimiter)
while threadData.shared.buffered and (
threadData.shared.lastFlushed + 1
>= threadData.shared.buffered[0][0]
or
len(threadData.shared.buffered) >
MAX_BUFFERED_PARTIAL_UNION_LENGTH):
threadData.shared.lastFlushed, _ = threadData.shared.buffered[
0]
if not isNoneValue(_):
threadData.shared.value.extend(
arrayizeValue(_))
del threadData.shared.buffered[0]
if conf.verbose == 1 and not (
threadData.resumed
and kb.suppressResumeInfo
) and not threadData.shared.showEta and not kb.bruteMode:
_ = ','.join("'%s'" % _ for _ in (
flattenValue(arrayizeValue(items)) if
not isinstance(items, six.string_types)
else [items]))
status = "[%s] [INFO] %s: %s" % (
time.strftime("%X"), "resumed"
if threadData.resumed else "retrieved",
_ if kb.safeCharEncode else
safecharencode(_))
if len(status) > width:
status = "%s..." % status[:width - 3]
dataToStdout("%s\n" % status)
def forgeHeaders(items=None, base=None):
"""
Prepare HTTP Cookie, HTTP User-Agent and HTTP Referer headers to use when performing
the HTTP requests
"""
items = items or {}
for _ in list(items.keys()):
if items[_] is None:
del items[_]
headers = OrderedDict(conf.httpHeaders if base is None else base)
headers.update(items.items())
class _str(str):
def capitalize(self):
return _str(self)
def title(self):
return _str(self)
_ = headers
headers = OrderedDict()
for key, value in _.items():
success = False
for _ in headers:
if _.upper() == key.upper():
del headers[_]
break
if key.upper() not in (
_.upper() for _ in getPublicTypeMembers(HTTP_HEADER, True)):
try:
headers[_str(
key
)] = value # dirty hack for http://bugs.python.org/issue12455
except UnicodeEncodeError: # don't do the hack on non-ASCII header names (they have to be properly encoded later on)
pass
else:
success = True
if not success:
key = '-'.join(_.capitalize() for _ in key.split('-'))
headers[key] = value
if conf.cj:
if HTTP_HEADER.COOKIE in headers:
for cookie in conf.cj:
if cookie.domain_specified and not (
conf.hostname or "").endswith(cookie.domain):
continue
if ("%s=" % getUnicode(cookie.name)) in getUnicode(
headers[HTTP_HEADER.COOKIE]):
if conf.loadCookies:
conf.httpHeaders = filterNone(
(item if item[0] != HTTP_HEADER.COOKIE else None)
for item in conf.httpHeaders)
elif kb.mergeCookies is None:
message = "you provided a HTTP %s header value, while " % HTTP_HEADER.COOKIE
message += "target URL provides its own cookies within "
message += "HTTP %s header which intersect with yours. " % HTTP_HEADER.SET_COOKIE
message += "Do you want to merge them in further requests? [Y/n] "
kb.mergeCookies = readInput(message,
default='Y',
boolean=True)
if kb.mergeCookies and kb.injection.place != PLACE.COOKIE:
def _(value):
return re.sub(
r"(?i)\b%s=[^%s]+" %
(re.escape(getUnicode(cookie.name)),
conf.cookieDel or DEFAULT_COOKIE_DELIMITER),
("%s=%s" % (getUnicode(cookie.name),
getUnicode(cookie.value))).replace(
'\\', r'\\'), value)
headers[HTTP_HEADER.COOKIE] = _(
headers[HTTP_HEADER.COOKIE])
if PLACE.COOKIE in conf.parameters:
conf.parameters[PLACE.COOKIE] = _(
conf.parameters[PLACE.COOKIE])
conf.httpHeaders = [
(item[0], item[1]
if item[0] != HTTP_HEADER.COOKIE else _(item[1]))
for item in conf.httpHeaders
]
elif not kb.testMode:
headers[HTTP_HEADER.COOKIE] += "%s %s=%s" % (
conf.cookieDel or DEFAULT_COOKIE_DELIMITER,
getUnicode(cookie.name), getUnicode(cookie.value))
if kb.testMode and not any((conf.csrfToken, conf.safeUrl)):
resetCookieJar(conf.cj)
return headers
def log_event(event_tuple, packet=None, skip_write=False, skip_condensing=False):
global _condensing_thread
if _condensing_thread is None:
_condensing_thread = threading.Thread(target=flush_condensed_events)
_condensing_thread.daemon = True
_condensing_thread.start()
try:
sec, usec, src_ip, src_port, dst_ip, dst_port, proto, trail_type, trail, info, reference = event_tuple
if ignore_event(event_tuple):
return
if not (any(check_whitelisted(_) for _ in (src_ip, dst_ip)) and trail_type != TRAIL.DNS): # DNS requests/responses can't be whitelisted based on src_ip/dst_ip
if not skip_write:
localtime = "%s.%06d" % (time.strftime(TIME_FORMAT, time.localtime(int(sec))), usec)
if not skip_condensing:
if any(_ in info for _ in CONDENSE_ON_INFO_KEYWORDS):
with _condensing_lock:
key = (src_ip, trail)
if key not in _condensed_events:
_condensed_events[key] = []
_condensed_events[key].append(event_tuple)
return
current_bucket = sec // config.PROCESS_COUNT
if getattr(_thread_data, "log_bucket", None) != current_bucket: # log throttling
_thread_data.log_bucket = current_bucket
_thread_data.log_trails = set()
else:
if any(_ in _thread_data.log_trails for _ in ((src_ip, trail), (dst_ip, trail))):
return
else:
_thread_data.log_trails.add((src_ip, trail))
_thread_data.log_trails.add((dst_ip, trail))
event = "%s %s %s\n" % (safe_value(localtime), safe_value(config.SENSOR_NAME), " ".join(safe_value(_) for _ in event_tuple[2:]))
if not config.DISABLE_LOCAL_LOG_STORAGE:
handle = get_event_log_handle(sec)
os.write(handle, event.encode(UNICODE_ENCODING))
if config.LOG_SERVER:
if config.LOG_SERVER.count(':') > 1:
remote_host, remote_port = config.LOG_SERVER.replace('[', '').replace(']', '').rsplit(':', 1)
# Reference: https://github.com/squeaky-pl/zenchmarks/blob/master/vendor/twisted/internet/tcp.py
_AI_NUMERICSERV = getattr(socket, "AI_NUMERICSERV", 0)
_NUMERIC_ONLY = socket.AI_NUMERICHOST | _AI_NUMERICSERV
_address = socket.getaddrinfo(remote_host, int(remote_port) if str(remote_port or "").isdigit() else 0, 0, 0, 0, _NUMERIC_ONLY)[0][4]
else:
remote_host, remote_port = config.LOG_SERVER.split(':')
_address = (remote_host, int(remote_port))
s = socket.socket(socket.AF_INET if len(_address) == 2 else socket.AF_INET6, socket.SOCK_DGRAM)
s.sendto(("%s %s" % (sec, event)).encode(UNICODE_ENCODING), _address)
if config.SYSLOG_SERVER or config.LOGSTASH_SERVER:
severity = "medium"
if config.REMOTE_SEVERITY_REGEX:
match = re.search(config.REMOTE_SEVERITY_REGEX, info)
if match:
for _ in ("low", "medium", "high"):
if match.group(_):
severity = _
break
if config.SYSLOG_SERVER:
extension = "src=%s spt=%s dst=%s dpt=%s trail=%s ref=%s" % (src_ip, src_port, dst_ip, dst_port, trail, reference)
_ = CEF_FORMAT.format(syslog_time=time.strftime("%b %d %H:%M:%S", time.localtime(int(sec))), host=HOSTNAME, device_vendor=NAME, device_product="sensor", device_version=VERSION, signature_id=time.strftime("%Y-%m-%d", time.localtime(os.path.getctime(config.TRAILS_FILE))), name=info, severity={"low": 0, "medium": 1, "high": 2}.get(severity), extension=extension)
remote_host, remote_port = config.SYSLOG_SERVER.split(':')
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.sendto(_.encode(UNICODE_ENCODING), (remote_host, int(remote_port)))
if config.LOGSTASH_SERVER:
_ = OrderedDict((("timestamp", sec), ("sensor", HOSTNAME), ("severity", severity), ("src_ip", src_ip), ("src_port", src_port), ("dst_ip", dst_ip), ("dst_port", dst_port), ("proto", proto), ("type", trail_type), ("trail", trail), ("info", info), ("reference", reference)))
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
remote_host, remote_port = config.LOGSTASH_SERVER.split(':')
s.sendto(json.dumps(_).encode(UNICODE_ENCODING), (remote_host, int(remote_port)))
if (config.DISABLE_LOCAL_LOG_STORAGE and not any((config.LOG_SERVER, config.SYSLOG_SERVER))) or config.console:
sys.stderr.write(event)
sys.stderr.flush()
if config.plugin_functions:
for _ in config.plugin_functions:
_(event_tuple, packet)
except (OSError, IOError):
if config.SHOW_DEBUG:
traceback.print_exc()
def forgeHeaders(items=None, base=None):
"""
Prepare HTTP Cookie, HTTP User-Agent and HTTP Referer headers to use when performing
the HTTP requests
"""
items = items or {}
for _ in list(items.keys()):
if items[_] is None:
del items[_]
headers = OrderedDict(conf.httpHeaders if base is None else base)
headers.update(items.items())
class _str(str):
def capitalize(self):
return _str(self)
def title(self):
return _str(self)
_ = headers
headers = OrderedDict()
for key, value in _.items():
success = False
for _ in headers:
if _.upper() == key.upper():
del headers[_]
break
if key.upper() not in (_.upper() for _ in getPublicTypeMembers(HTTP_HEADER, True)):
try:
headers[_str(key)] = value # dirty hack for http://bugs.python.org/issue12455
except UnicodeEncodeError: # don't do the hack on non-ASCII header names (they have to be properly encoded later on)
pass
else:
success = True
if not success:
key = '-'.join(_.capitalize() for _ in key.split('-'))
headers[key] = value
if conf.cj:
if HTTP_HEADER.COOKIE in headers:
for cookie in conf.cj:
if cookie.domain_specified and not (conf.hostname or "").endswith(cookie.domain):
continue
if ("%s=" % getUnicode(cookie.name)) in getUnicode(headers[HTTP_HEADER.COOKIE]):
if conf.loadCookies:
conf.httpHeaders = filterNone((item if item[0] != HTTP_HEADER.COOKIE else None) for item in conf.httpHeaders)
elif kb.mergeCookies is None:
message = "you provided a HTTP %s header value. " % HTTP_HEADER.COOKIE
message += "The target URL provided its own cookies within "
message += "the HTTP %s header which intersect with yours. " % HTTP_HEADER.SET_COOKIE
message += "Do you want to merge them in further requests? [Y/n] "
kb.mergeCookies = readInput(message, default='Y', boolean=True)
if kb.mergeCookies and kb.injection.place != PLACE.COOKIE:
def _(value):
return re.sub(r"(?i)\b%s=[^%s]+" % (re.escape(getUnicode(cookie.name)), conf.cookieDel or DEFAULT_COOKIE_DELIMITER), ("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value))).replace('\\', r'\\'), value)
headers[HTTP_HEADER.COOKIE] = _(headers[HTTP_HEADER.COOKIE])
if PLACE.COOKIE in conf.parameters:
conf.parameters[PLACE.COOKIE] = _(conf.parameters[PLACE.COOKIE])
conf.httpHeaders = [(item[0], item[1] if item[0] != HTTP_HEADER.COOKIE else _(item[1])) for item in conf.httpHeaders]
elif not kb.testMode:
headers[HTTP_HEADER.COOKIE] += "%s %s=%s" % (conf.cookieDel or DEFAULT_COOKIE_DELIMITER, getUnicode(cookie.name), getUnicode(cookie.value))
if kb.testMode and not any((conf.csrfToken, conf.safeUrl)):
resetCookieJar(conf.cj)
return headers
def unionThread():
threadData = getCurrentThreadData()
while kb.threadContinue:
with kb.locks.limit:
try:
threadData.shared.counter += 1
num = next(threadData.shared.limits)
except StopIteration:
break
if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE):
field = expressionFieldsList[0]
elif Backend.isDbms(DBMS.ORACLE):
field = expressionFieldsList
else:
field = None
limitedExpr = agent.limitQuery(num, expression, field)
output = _oneShotUnionUse(limitedExpr, unpack, True)
if not kb.threadContinue:
break
if output:
with kb.locks.value:
if all(_ in output for _ in (kb.chars.start, kb.chars.stop)):
items = parseUnionPage(output)
if threadData.shared.showEta:
threadData.shared.progress.progress(threadData.shared.counter)
if isListLike(items):
# in case that we requested N columns and we get M!=N then we have to filter a bit
if len(items) > 1 and len(expressionFieldsList) > 1:
items = [item for item in items if isListLike(item) and len(item) == len(expressionFieldsList)]
items = [_ for _ in flattenValue(items)]
if len(items) > len(expressionFieldsList):
filtered = OrderedDict()
for item in items:
key = re.sub(r"[^A-Za-z0-9]", "", item).lower()
if key not in filtered or re.search(r"[^A-Za-z0-9]", item):
filtered[key] = item
items = filtered.values()
items = [items]
index = None
for index in xrange(1 + len(threadData.shared.buffered)):
if index < len(threadData.shared.buffered) and threadData.shared.buffered[index][0] >= num:
break
threadData.shared.buffered.insert(index or 0, (num, items))
else:
index = None
if threadData.shared.showEta:
threadData.shared.progress.progress(threadData.shared.counter)
for index in xrange(1 + len(threadData.shared.buffered)):
if index < len(threadData.shared.buffered) and threadData.shared.buffered[index][0] >= num:
break
threadData.shared.buffered.insert(index or 0, (num, None))
items = output.replace(kb.chars.start, "").replace(kb.chars.stop, "").split(kb.chars.delimiter)
while threadData.shared.buffered and (threadData.shared.lastFlushed + 1 >= threadData.shared.buffered[0][0] or len(threadData.shared.buffered) > MAX_BUFFERED_PARTIAL_UNION_LENGTH):
threadData.shared.lastFlushed, _ = threadData.shared.buffered[0]
if not isNoneValue(_):
threadData.shared.value.extend(arrayizeValue(_))
del threadData.shared.buffered[0]
if conf.verbose == 1 and not (threadData.resumed and kb.suppressResumeInfo) and not threadData.shared.showEta:
_ = ','.join("'%s'" % _ for _ in (flattenValue(arrayizeValue(items)) if not isinstance(items, six.string_types) else [items]))
status = "[%s] [INFO] %s: %s" % (time.strftime("%X"), "resumed" if threadData.resumed else "retrieved", _ if kb.safeCharEncode else safecharencode(_))
if len(status) > width:
status = "%s..." % status[:width - 3]
dataToStdout("%s\n" % status)
