def _web_parsing_callback(parsed_page: dict):
"""
The callback function of crawler.
This method writes the parsed pages in a json file
:param parsed_page: The parsed page
"""
JsonSerializer.add_item_to_dict(parsed_page.get('url'), parsed_page, job.json_file)
def _deep_inject_form(href, depth=1):
# Check the domain
if href in parsed_forms or \
urlparse(href).netloc != base_url or \
(max_depth is not None and depth > max_depth):
return ''
# Visit the current href
parsed_relevant, request_cookies = HtmlParser.relevant_parse(href)
parsed_forms[href] = HtmlParser.find_forms(parsed_relevant, href)
# Find adjacent links
links = HtmlParser.find_links(parsed_relevant)
if len(parsed_forms) % 10 == 0:
Log.info('Writing result in ' + out_file + '...')
JsonSerializer.set_dictionary(parsed_forms, out_file)
# Visit adjacent links
for link in links:
# print('link: '+link)
child_request_cookies = _deep_inject_form(link, depth + 1)
if len(child_request_cookies) > len(request_cookies):
request_cookies = child_request_cookies
return request_cookies
def _sniffer_callback(pkt: dict):
"""
The callback function of packet sniffer.
This method writes the sniffed packets in a json file
:param pkt: The sniffed packet
"""
JsonSerializer.add_item_to_dict(pkt['number'], pkt, job.json_file)
def __add__(key, element):
dictionary = JsonSerializer.get_dictionary(APP_SETTINGS)
elements = dictionary.get(key)
if type(elements) != list:
elements = []
if element not in elements:
elements.append(element)
dictionary[key] = elements
JsonSerializer.set_dictionary(dictionary, APP_SETTINGS)
return True
def _update_manufacturer_dict(self):
manufacturer_response = HttpRequest.request(
MacManufacturer._MANUFACTURERS_URL)
if manufacturer_response is None:
return
if manufacturer_response.text is None:
return
manufacturer_dict = dict()
manufacturer_list = manufacturer_response.text.splitlines()
for manufacturer in manufacturer_list:
if len(manufacturer) < 1:
continue
if manufacturer[0] == '#':
continue
manufacturer_details = manufacturer.split('\t')
i = 0
mac = None
lookup_dict = {
MacManufacturer._MANUFACTURERS_DETAIL_DICT[1]: None,
MacManufacturer._MANUFACTURERS_DETAIL_DICT[2]: None,
MacManufacturer._MANUFACTURERS_DETAIL_DICT[3]: None
}
for detail in manufacturer_details:
if detail == '':
continue
if i == 0:
# MAC address
mac_detail = detail.split('/')
if len(mac_detail) == 2:
# The mac has a sub mask, so the dict key is the first n bits
sub_mask = int(mac_detail[1]) / 4
mac_sub_mask = floor(sub_mask + (sub_mask / 2))
mac = mac_detail[0][0:mac_sub_mask]
elif len(mac_detail) == 1:
# The mac has not a sub mask
mac = mac_detail[0]
else:
Log.error("Wrong mac address: " + str(detail))
break
if i >= len(MacManufacturer._MANUFACTURERS_DETAIL_DICT):
Log.error("Wrong manufacturer details: " +
str(manufacturer_details))
break
lookup_dict[
MacManufacturer._MANUFACTURERS_DETAIL_DICT[i]] = detail
i += 1
if mac is None:
Log.error("Wrong manufacturer details: " +
str(manufacturer_details))
continue
manufacturer_dict[mac] = lookup_dict
if len(manufacturer_dict) > 0:
self._manufacturer_dict = manufacturer_dict
JsonSerializer.set_dictionary(self._manufacturer_dict,
MacManufacturer._MANUFACTURERS_JSON)
def deep_inject_form(url, max_depth, listen: bool = False) -> dict:
"""
Search a form in the page returned by url.
If it doesn't find a form, or the injection can't be done, it visit the website in search for other forms
:param listen: True if this method should listen and print the SQL tasks, otherwise False
:param url: str The url to visit
:param max_depth: int The max depth during the visit
:return A dictionary of SQL injection tasks
"""
base_url = urlparse(url).netloc
parsed_forms = dict()
out_file = APP_STORAGE_OUT + '/' + now(
) + '_DEEP_FORMS_' + base_url + '.json'
def _deep_inject_form(href, depth=1):
# Check the domain
if href in parsed_forms or \
urlparse(href).netloc != base_url or \
(max_depth is not None and depth > max_depth):
return ''
# Visit the current href
parsed_relevant, request_cookies = HtmlParser.relevant_parse(href)
# Find forms in page
parsed_forms[href] = HtmlParser.find_forms(parsed_relevant, href)
# Find adjacent links
links = HtmlParser.find_links(parsed_relevant)
if len(parsed_forms) % 10 == 0:
Log.info('Writing result in ' + out_file + '...')
JsonSerializer.set_dictionary(parsed_forms, out_file)
# Visit adjacent links
for link in links:
# print('link: '+link)
child_request_cookies = _deep_inject_form(link, depth + 1)
if len(child_request_cookies) > len(request_cookies):
request_cookies = child_request_cookies
return request_cookies
cookies = _deep_inject_form(url)
Log.info('Writing result in ' + out_file + '...')
JsonSerializer.set_dictionary(parsed_forms, out_file)
Log.success('Result wrote in ' + out_file)
Log.success('Website crawled! Found ' + str(len(parsed_forms)) +
' pages')
tasks = SqlmapClient.try_inject_forms(parsed_forms, cookies)
if listen:
SqlInjection.__listen_tasks(tasks)
return tasks
def __remove__(key, element):
dictionary = JsonSerializer.get_dictionary(APP_SETTINGS)
if dictionary.get(key) is None:
return True
if element == '*':
# Rimuove tutti gli elementi
dictionary[key] = []
JsonSerializer.set_dictionary(dictionary, APP_SETTINGS)
return True
elements = dictionary[key]
if element in elements:
elements.remove(element)
dictionary[key] = elements
JsonSerializer.set_dictionary(dictionary, APP_SETTINGS)
return True
def json_dict(self) -> dict:
attempts = 0
json_dict = JsonSerializer.get_dictionary(self.json_file)
while len(json_dict) == 0:
sleep(0.2)
# Prevents parallel access errors to file
json_dict = JsonSerializer.get_dictionary(self.json_file)
attempts += 1
if attempts >= 5:
break
return sort_dict(dict(sorted(
json_dict.items(),
key=lambda e: int(e[1]['number']),
reverse=True
)))
def request(url: str,
request_type: str = HttpRequest.Type.GET,
json: dict or list = None) -> dict:
"""
Send a request to sqlmap-api server and then load the data json as dict
:param url: The sqlmap-api url (eg. "http://127.0.0.1:8775/task/new")
:param request_type: get|post|put|patch|delete
:param json: The json to send
:rtype: dict
"""
response = HttpRequest.request(url, request_type, json=json)
r_data = JsonSerializer.load_json(response.text)
Log.info('Response data of ' + url + ': ' + str(r_data))
if not r_data['success']:
Log.error('Response data of ' + url + ' has { success: False }')
raise requests.RequestException('Request to ' + url + ' failed')
return r_data
def _crawl(href: str, curr_depth: int = 0):
parsed_href = urlparse(href)
href = parsed_href.scheme + '://' + parsed_href.netloc + parsed_href.path
if parsed_href.query != '':
href += '?' + parsed_href.query
if href in parsed_urls or \
parsed_href.netloc not in base_urls or \
'mailto:' in parsed_href.path or \
(0 <= depth and (depth < curr_depth)):
return
# Visit the current href
if parsing_type == HtmlParser.TYPE_ALL:
parsed, _ = HtmlParser.all_parse(href, cookies=cookies)
else:
parsed, _ = HtmlParser.relevant_parse(href, cookies=cookies)
parsed_hash = hash(JsonSerializer.dumps_json(parsed))
if parsed_hash in parsed_hashes:
return
parsed_hashes.add(parsed_hash)
parsed_urls.add(href)
if parsing_type == HtmlParser.TYPE_FORM:
# Find forms in page
parsed_page = HtmlParser.find_forms(parsed, href)
elif parsing_type == HtmlParser.TYPE_META:
# Find metadata in page
parsed_page = HtmlParser.find_meta(parsed)
else:
parsed_page = parsed
if parsed_page.get('tag') is not None:
parsed_page = {0: parsed_page}
parsed_page['url'] = href
callback(parsed_page)
# Find adjacent links
links = HtmlParser.find_links(parsed)
for link in links:
_crawl(link, curr_depth + 1)
def __set__(key, value):
dictionary = JsonSerializer.get_dictionary(APP_SETTINGS)
dictionary[key] = value
JsonSerializer.set_dictionary(dictionary, APP_SETTINGS)
return True
def __get__(key=None):
dictionary = JsonSerializer.get_dictionary(APP_SETTINGS)
if key is None:
return dictionary
return dictionary.get(key)
def __init__(self):
self._manufacturer_dict = JsonSerializer.get_dictionary(
MacManufacturer._MANUFACTURERS_JSON)
self._update_manufacturer_dict()