def unionThread():
threadData = getCurrentThreadData()
while kb.threadContinue:
with kb.locks.limit:
try:
valueStart = time.time()
threadData.shared.counter += 1
num = threadData.shared.limits.next()
except StopIteration:
break
if Backend.getIdentifiedDbms() in (DBMS.MSSQL,
DBMS.SYBASE):
field = expressionFieldsList[0]
elif Backend.isDbms(DBMS.ORACLE):
field = expressionFieldsList
else:
field = None
limitedExpr = agent.limitQuery(
num, expression, field)
output = _oneShotUnionUse(limitedExpr, unpack,
True)
if not kb.threadContinue:
break
if output:
with kb.locks.value:
if all(_ in output
for _ in (kb.chars.start,
kb.chars.stop)):
items = parseUnionPage(output)
if threadData.shared.showEta:
threadData.shared.progress.progress(
threadData.shared.counter)
if isListLike(items):
# in case that we requested N columns and we get M!=N then we have to filter a bit
if len(items) > 1 and len(
expressionFieldsList) > 1:
items = [
item for item in items
if isListLike(item)
and len(item) == len(
expressionFieldsList)
]
items = [
_ for _ in flattenValue(items)
]
if len(items) > len(
expressionFieldsList):
filtered = OrderedDict()
for item in items:
key = re.sub(
r"[^A-Za-z0-9]", "",
item).lower()
if key not in filtered or re.search(
r"[^A-Za-z0-9]",
item):
filtered[key] = item
items = filtered.values()
items = [items]
index = None
for index in xrange(1 + len(
threadData.shared.buffered)):
if index < len(
threadData.shared.buffered
) and threadData.shared.buffered[
index][0] >= num:
break
threadData.shared.buffered.insert(
index or 0, (num, items))
else:
index = None
if threadData.shared.showEta:
threadData.shared.progress.progress(
threadData.shared.counter)
for index in xrange(1 + len(
threadData.shared.buffered)):
if index < len(
threadData.shared.buffered
) and threadData.shared.buffered[
index][0] >= num:
break
threadData.shared.buffered.insert(
index or 0, (num, None))
items = output.replace(
kb.chars.start, "").replace(
kb.chars.stop,
"").split(kb.chars.delimiter)
while threadData.shared.buffered and (
threadData.shared.lastFlushed + 1
>= threadData.shared.buffered[0][0]
or
len(threadData.shared.buffered) >
MAX_BUFFERED_PARTIAL_UNION_LENGTH):
threadData.shared.lastFlushed, _ = threadData.shared.buffered[
0]
if not isNoneValue(_):
threadData.shared.value.extend(
arrayizeValue(_))
del threadData.shared.buffered[0]
if conf.verbose == 1 and not (
threadData.resumed
and kb.suppressResumeInfo
) and not threadData.shared.showEta:
_ = ','.join(
"\"%s\"" % _ for _ in
flattenValue(arrayizeValue(
items))) if not isinstance(
items, basestring) else items
status = "[%s] [INFO] %s: %s" % (
time.strftime("%X"), "resumed"
if threadData.resumed else "retrieved",
_ if kb.safeCharEncode else
safecharencode(_))
if len(status) > width:
status = "%s..." % status[:width - 3]
dataToStdout("%s\n" % status)
def unionThread():
threadData = getCurrentThreadData()
while kb.threadContinue:
with kb.locks.limit:
try:
valueStart = time.time()
threadData.shared.counter += 1
num = threadData.shared.limits.next()
except StopIteration:
break
if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE):
field = expressionFieldsList[0]
elif Backend.isDbms(DBMS.ORACLE):
field = expressionFieldsList
else:
field = None
limitedExpr = agent.limitQuery(num, expression, field)
output = _oneShotUnionUse(limitedExpr, unpack, True)
if not kb.threadContinue:
break
if output:
with kb.locks.value:
if all(_ in output for _ in (kb.chars.start, kb.chars.stop)):
items = parseUnionPage(output)
if threadData.shared.showEta:
threadData.shared.progress.progress(time.time() - valueStart, threadData.shared.counter)
if isListLike(items):
# in case that we requested N columns and we get M!=N then we have to filter a bit
if len(items) > 1 and len(expressionFieldsList) > 1:
items = [item for item in items if isListLike(item) and len(item) == len(expressionFieldsList)]
items = [_ for _ in flattenValue(items)]
if len(items) > len(expressionFieldsList):
filtered = OrderedDict()
for item in items:
key = re.sub(r"[^A-Za-z0-9]", "", item).lower()
if key not in filtered or re.search(r"[^A-Za-z0-9]", item):
filtered[key] = item
items = filtered.values()
items = [items]
index = None
for index in xrange(1 + len(threadData.shared.buffered)):
if index < len(threadData.shared.buffered) and threadData.shared.buffered[index][0] >= num:
break
threadData.shared.buffered.insert(index or 0, (num, items))
else:
index = None
if threadData.shared.showEta:
threadData.shared.progress.progress(time.time() - valueStart, threadData.shared.counter)
for index in xrange(1 + len(threadData.shared.buffered)):
if index < len(threadData.shared.buffered) and threadData.shared.buffered[index][0] >= num:
break
threadData.shared.buffered.insert(index or 0, (num, None))
items = output.replace(kb.chars.start, "").replace(kb.chars.stop, "").split(kb.chars.delimiter)
while threadData.shared.buffered and (threadData.shared.lastFlushed + 1 >= threadData.shared.buffered[0][0] or len(threadData.shared.buffered) > MAX_BUFFERED_PARTIAL_UNION_LENGTH):
threadData.shared.lastFlushed, _ = threadData.shared.buffered[0]
if not isNoneValue(_):
threadData.shared.value.extend(arrayizeValue(_))
del threadData.shared.buffered[0]
if conf.verbose == 1 and not (threadData.resumed and kb.suppressResumeInfo) and not threadData.shared.showEta:
_ = ','.join("\"%s\"" % _ for _ in flattenValue(arrayizeValue(items))) if not isinstance(items, basestring) else items
status = "[%s] [INFO] %s: %s" % (time.strftime("%X"), "resumed" if threadData.resumed else "retrieved", _ if kb.safeCharEncode else safecharencode(_))
if len(status) > width:
status = "%s..." % status[:width - 3]
dataToStdout("%s\n" % status)
