def _send_mutants_in_threads(self, func, iterable, callback, **kwds):
"""
Please note that this method blocks from the caller's point of view
but performs all the HTTP requests in parallel threads.
"""
imap_unordered = self.worker_pool.imap_unordered
awre = apply_with_return_error
# You can use this code to debug issues that happen in threads, by
# simply not using them:
#
# for i in iterable:
# callback(i, func(i))
# return
#
# Now the real code:
func = return_args(func, **kwds)
args = zip(repeat(func), iterable)
for result in imap_unordered(awre, args):
# re-raise the thread exception in the main thread with this method
# so we get a nice traceback instead of things like the ones we see
# in https://github.com/andresriancho/w3af/issues/7286
if isinstance(result, Error):
result.reraise()
else:
(mutant,), http_response = result
callback(mutant, http_response)
def get_data(self, data_len):
"""
:param data_len: The data length to retrieve
:return: A string with the XML data!
HTTP library exceptions are not handled in order to make the code
clearer.
"""
data = [None] * data_len
mod_get_char = return_args(self.get_char_in_pos)
imap_unordered = self.worker_pool.imap_unordered
len_iter = xrange(data_len)
for (pos, ), char in imap_unordered(mod_get_char, len_iter):
data[pos] = char
clean_data = []
current = ''
for char in data:
if char is None:
current = current.strip()
if current != '':
clean_data.append(current)
current = ''
else:
current += char
return '\n'.join(clean_data)
def _send_mutants_in_threads(self, func, iterable, callback, **kwds):
"""
Please note that this method blocks from the caller's point of view
but performs all the HTTP requests in parallel threads.
"""
imap_unordered = self.worker_pool.imap_unordered
awre = apply_with_return_error
# You can use this code to debug issues that happen in threads, by
# simply not using them:
#
# for i in iterable:
# callback(i, func(i))
# return
#
# Now the real code:
func = return_args(func, **kwds)
args = zip(repeat(func), iterable)
for result in imap_unordered(awre, args):
# re-raise the thread exception in the main thread with this method
# so we get a nice traceback instead of things like the ones we see
# in https://github.com/andresriancho/w3af/issues/7286
if isinstance(result, Error):
result.reraise()
else:
(mutant,), http_response = result
callback(mutant, http_response)
def get_data(self, data_len):
"""
:param data_len: The data length to retrieve
:return: A string with the XML data!
HTTP library exceptions are not handled in order to make the code
clearer.
"""
data = [None] * data_len
mod_get_char = return_args(self.get_char_in_pos)
imap_unordered = self.worker_pool.imap_unordered
len_iter = xrange(data_len)
for (pos,), char in imap_unordered(mod_get_char, len_iter):
data[pos] = char
clean_data = []
current = ''
for char in data:
if char is None:
current = current.strip()
if current != '':
clean_data.append(current)
current = ''
else:
current += char
return '\n'.join(clean_data)
def _consume(self, function_id, work_unit):
# self._consumer_plugins init the crawl_infra's param comment by Lightning
for plugin in self._consumer_plugins:
print 'crawl_infra _comsume plugin *********' + str(plugin)
if not self._running:
return
if plugin in self._disabled_plugins:
continue
om.out.debug('%s plugin is testing: "%s"' %
(plugin.get_name(), work_unit))
# start the disk monitor
self._run_observers(work_unit)
# TODO: unittest what happens if an exception (which is not handled
# by the exception handler) is raised. Who's doing a .get()
# on those ApplyResults generated here?
# run crawl and infrastructure plugins.
# callback the self._plugin_finished_cb after self._discover_worker
# done .comment by Lightning
self._threadpool.apply_async(return_args(self._discover_worker), (
plugin,
work_unit,
),
callback=self._plugin_finished_cb)
# pylint: disable=E1120
self._route_all_plugin_results()
def _consume(self, function_id, work_unit):
self._run_observers(work_unit)
for plugin in self._consumer_plugins:
self._threadpool.apply_async(return_args(self._bruteforce),
(plugin, work_unit,),
callback=self._plugin_finished_cb)
def _consume(self, function_id, work_unit):
self._run_observers(work_unit)
for plugin in self._consumer_plugins:
self._threadpool.apply_async(return_args(self._bruteforce),
(plugin, work_unit,),
callback=self._plugin_finished_cb)
def _send_in_threads(self, base_url, vhosts):
base_url_repeater = repeat(base_url)
args_iterator = izip(base_url_repeater, vhosts)
http_get = return_args(one_to_many(self._http_get_vhost))
pool_results = self.worker_pool.imap_unordered(http_get, args_iterator)
for ((base_url, vhost),), vhost_response in pool_results:
yield vhost, vhost_response
def read_multi(self, fname_iter):
"""
:param fname_iter: An iterator that yields all the file names to read.
"""
read_file = return_args(self.shell.read)
results = self.worker_pool.imap_unordered(read_file, fname_iter)
for (file_name,), content in results:
yield file_name, content
def _send_in_threads(self, base_url, vhosts):
base_url_repeater = repeat(base_url)
args_iterator = izip(base_url_repeater, vhosts)
http_get = return_args(one_to_many(self._http_get_vhost))
pool_results = self.worker_pool.imap_unordered(http_get, args_iterator)
for ((base_url, vhost), ), vhost_response in pool_results:
yield vhost, vhost_response
def read_multi(self, fname_iter):
"""
:param fname_iter: An iterator that yields all the file names to read.
"""
read_file = return_args(self.shell.read)
results = self.worker_pool.imap_unordered(read_file, fname_iter)
for (file_name, ), content in results:
yield file_name, content
def _consume(self, work_unit):
for plugin in self._consumer_plugins:
stats = '%s plugin is testing: "%s"'
om.out.debug(stats % (plugin.get_name(), work_unit))
self._threadpool.apply_async(return_args(self._bruteforce),
(plugin, work_unit,),
callback=self._plugin_finished_cb)
def read_multi(self, fname_iter, error_handler=None):
"""
:param fname_iter: An iterator that yields all the file names to read.
:param error_handler: A function which creates a wrapper around the
function that it takes as parameter, returning
the same tuple as the read_multi function.
:yield: Tuple:
- The name of the file which was read
- The contents of the file
"""
if error_handler is None:
read_file = return_args(self.shell.read)
else:
read_file = return_args(error_handler(self.shell.read))
results = self.worker_pool.imap_unordered(read_file, fname_iter)
for (file_name, ), content in results:
yield file_name, content
def read_multi(self, fname_iter, error_handler=None):
"""
:param fname_iter: An iterator that yields all the file names to read.
:param error_handler: A function which creates a wrapper around the
function that it takes as parameter, returning
the same tuple as the read_multi function.
:yield: Tuple:
- The name of the file which was read
- The contents of the file
"""
if error_handler is None:
read_file = return_args(self.shell.read)
else:
read_file = return_args(error_handler(self.shell.read))
results = self.worker_pool.imap_unordered(read_file, fname_iter)
for (file_name,), content in results:
yield file_name, content
def _send_mutants_in_threads(self, func, iterable, callback, **kwds):
"""
Please note that this method blocks from the caller's point of view
but performs all the HTTP requests in parallel threads.
"""
func = return_args(func, **kwds)
imap_unordered = self.worker_pool.imap_unordered
for (mutant,), http_response in imap_unordered(func, iterable):
callback(mutant, http_response)
def _consume(self, function_id, work_unit):
for plugin in self._consumer_plugins:
stats = '%s plugin is testing: "%s"'
om.out.debug(stats % (plugin.get_name(), work_unit))
self._threadpool.apply_async(return_args(self._bruteforce), (
plugin,
work_unit,
),
callback=self._plugin_finished_cb)
def api_read(self):
self.result = {}
self.result['bad_kernel_modules'] = []
self.result['backdoor_files'] = []
self.k = 400
self._check_kernel_modules()
read_file = return_args(self._read_with_progress)
fname_iter = self.fname_generator()
for (file_name,), content in self.worker_pool.imap_unordered(read_file, fname_iter):
if content:
self.result['backdoor_files'].append(file_name)
return self.result
def _send_mutants_in_threads(self, func, iterable, callback, **kwds):
"""
Please note that this method blocks from the caller's point of view
but performs all the HTTP requests in parallel threads.
"""
# You can use this code to debug issues that happen in threads, by
# simply not using them:
#
# for i in iterable:
# callback(i, func(i))
# return
#
# Now the real code:
func = return_args(func, **kwds)
imap_unordered = self.worker_pool.imap_unordered
for (mutant,), http_response in imap_unordered(func, iterable):
callback(mutant, http_response)
def _consume(self, work_unit):
for plugin in self._consumer_plugins:
if not self._running:
return
if plugin in self._disabled_plugins:
continue
om.out.debug('%s plugin is testing: "%s"' % (plugin.get_name(), work_unit))
# TODO: unittest what happens if an exception (which is not handled
# by the exception handler) is raised. Who's doing a .get()
# on those ApplyResults generated here?
self._threadpool.apply_async(
return_args(self._discover_worker), (plugin, work_unit), callback=self._plugin_finished_cb
)
self._route_all_plugin_results()
def _send_mutants_in_threads(self, func, iterable, callback, **kwds):
"""
Please note that this method blocks from the caller's point of view
but performs all the HTTP requests in parallel threads.
"""
# You can use this code to debug issues that happen in threads, by
# simply not using them:
#
# for i in iterable:
# callback(i, func(i))
# return
#
# Now the real code:
func = return_args(func, **kwds)
imap_unordered = self.worker_pool.imap_unordered
for (mutant, ), http_response in imap_unordered(func, iterable):
callback(mutant, http_response)
def _send_mutants_in_threads(self, func, iterable, callback, **kwds):
"""
Please note that this method blocks from the caller's point of view
but performs all the HTTP requests in parallel threads.
:param func: The function to use to send the mutants
:param iterable: A list with the mutants
:param callback: A callable to invoke after each mutant is sent
"""
imap_unordered = self.worker_pool.imap_unordered
awre = apply_with_return_error
try:
num_tasks = len(iterable)
except TypeError:
# When the iterable is a python iterator which doesn't implement
# the __len__, then we don't know the number of received tasks
pass
else:
debugging_id = kwds.get('debugging_id', 'unknown')
msg = 'send_mutants_in_threads will send %s HTTP requests (did:%s)'
args = (num_tasks, debugging_id)
om.out.debug(msg % args)
# You can use this code to debug issues that happen in threads, by
# simply not using them:
#
# for i in iterable:
# callback(i, func(i))
# return
#
# Now the real code:
func = return_args(func, **kwds)
args = zip(repeat(func), iterable)
for result in imap_unordered(awre, args):
# re-raise the thread exception in the main thread with this method
# so we get a nice traceback instead of things like the ones we see
# in https://github.com/andresriancho/w3af/issues/7286
if isinstance(result, Error):
result.reraise()
else:
(mutant,), http_response = result
callback(mutant, http_response)
def _send_mutants_in_threads(self, func, iterable, callback, **kwds):
"""
Please note that this method blocks from the caller's point of view
but performs all the HTTP requests in parallel threads.
:param func: The function to use to send the mutants
:param iterable: A list with the mutants
:param callback: A callable to invoke after each mutant is sent
"""
imap_unordered = self.worker_pool.imap_unordered
awre = apply_with_return_error
try:
num_tasks = len(iterable)
except TypeError:
# When the iterable is a python iterator which doesn't implement
# the __len__, then we don't know the number of received tasks
pass
else:
debugging_id = kwds.get('debugging_id', 'unknown')
msg = 'send_mutants_in_threads will send %s HTTP requests (did:%s)'
args = (num_tasks, debugging_id)
om.out.debug(msg % args)
# You can use this code to debug issues that happen in threads, by
# simply not using them:
#
# for i in iterable:
# callback(i, func(i))
# return
#
# Now the real code:
func = return_args(func, **kwds)
args = zip(repeat(func), iterable)
for result in imap_unordered(awre, args):
# re-raise the thread exception in the main thread with this method
# so we get a nice traceback instead of things like the ones we see
# in https://github.com/andresriancho/w3af/issues/7286
if isinstance(result, Error):
result.reraise()
else:
(mutant, ), http_response = result
callback(mutant, http_response)
def _consume(self, function_id, work_unit):
for plugin in self._consumer_plugins:
if not self._running:
return
if plugin in self._disabled_plugins:
continue
self._run_observers(work_unit)
# TODO: unittest what happens if an exception (which is not handled
# by the exception handler) is raised. Who's doing a .get()
# on those ApplyResults generated here?
self._threadpool.apply_async(return_args(self._discover_worker),
(plugin, work_unit,),
callback=self._plugin_finished_cb)
# pylint: disable=E1120
self._route_all_plugin_results()
def _consume(self, work_unit):
for plugin in self._consumer_plugins:
if not self._running:
return
if plugin in self._disabled_plugins:
continue
om.out.debug('%s plugin is testing: "%s"' %
(plugin.get_name(), work_unit))
# TODO: unittest what happens if an exception (which is not handled
# by the exception handler) is raised. Who's doing a .get()
# on those ApplyResults generated here?
self._threadpool.apply_async(return_args(self._discover_worker), (
plugin,
work_unit,
),
callback=self._plugin_finished_cb)
self._route_all_plugin_results()
def test_basic(self):
args_int = return_args(int)
self.assertEqual((('3',), 3), args_int('3'))
def test_kwds(self):
args_int_two = return_args(int, base=2)
self.assertEqual((('1', ), 1), args_int_two('1'))
def test_two_params(self):
args_replace = return_args('foo123bar'.replace)
self.assertEqual((('123', ''), 'foobar'), args_replace('123', ''))
def test_basic(self):
args_int = return_args(int)
self.assertEqual((('3', ), 3), args_int('3'))
def test_two_params(self):
args_replace = return_args('foo123bar'.replace)
self.assertEqual((('123', ''), 'foobar'), args_replace('123', ''))
def test_kwds(self):
args_int_two = return_args(int, base=2)
self.assertEqual((('1',), 1), args_int_two('1'))
