def print_parsed(parsed: dict or list, depth: int = 0):
"""
Print The result of methods @parse and @relevant_parse (so a parsed html)
:param parsed: A parsed html
:param depth: Current depth to build a pretty tree
"""
space = ' ' * depth
if type(parsed) == dict:
print(space + '{')
for key, value in parsed.items():
if key == 'children':
HtmlParser.print_parsed(value, depth + 1)
elif is_listable(value):
print((space + ' ') + str(key) + ':')
HtmlParser.print_parsed(value, depth + 2)
# print((space+' ') + str(key) + ':')
# subspace = ' ' * (depth+1)
# for el in dict:
# if (is_listable(el)):
else:
print((space + ' ') + str(key) + ': ' + str(value))
print(space + '}')
elif type(parsed) == list:
for value in parsed:
HtmlParser.print_parsed(value, depth + 1)
else:
Log.error(str(parsed) + ' is not a valid parsed content!')
def server_to_defend(ip):
if APP_DEBUG:
Log.info('CALLED: Add.server_to_defend(' + str(ip) + ')')
if not validators.is_ip(ip):
Log.error(str(ip) + ' is not a valid ip address')
return False
return Add.__add__(keys.SERVER_TO_DEFEND, ip)
def find_links(parsed: dict or list) -> set:
"""
Search links inside a parsed html (dict)
:param parsed: A parsed html
:return: A set of found links
"""
links = set()
if parsed is None:
return links
if type(parsed) == dict:
attrs = parsed.get('attrs')
if attrs is not None:
url = None
if 'form' == parsed.get('tag'):
url = attrs.get('action')
elif 'a' == parsed.get('tag'):
url = attrs.get('href')
if url is not None:
links.add(url)
links = links.union(HtmlParser.find_links(parsed.get('children')))
elif type(parsed) == list:
for value in parsed:
links = links.union(HtmlParser.find_links(value))
else:
Log.error(str(parsed) + ' is not a valid parsed content!')
return links
def team_player(ip):
if APP_DEBUG:
Log.info('CALLED: Add.team_player(' + str(ip) + ')')
if not validators.is_ip(ip):
Log.error(str(ip) + ' is not a valid ip address')
return False
return Add.__add__(keys.TEAM_PLAYER, ip)
def server_to_defend(ip='*'):
if APP_DEBUG:
Log.info('CALLED: Remove.server_to_defend(' + str(ip) + ')')
if ip != '*' and not validators.is_ip(ip):
Log.error(str(ip) + ' is not a valid ip address')
return False
return Remove.__remove__(keys.SERVER_TO_DEFEND, ip)
def find_forms(parsed: dict or list, url=None) -> list:
"""
Search forms inside parsed html (dict)
:param parsed: A parsed html
:param url: The parsed url (or None)
:return: The list of found forms
"""
forms = []
if parsed is None:
return forms
if type(parsed) == dict:
if 'form' == parsed.get('tag'):
attrs = parsed.get('attrs')
action = attrs.get('action')
method = attrs.get('method')
if action is None:
action = url
if method is None:
method = HttpRequest.Type.POST
form = {
'method': method,
'action': action,
'inputs': HtmlParser.find_inputs(parsed.get('children'))
}
forms.append(form)
forms += HtmlParser.find_forms(parsed.get('children'), url)
elif type(parsed) == list:
for value in parsed:
forms += HtmlParser.find_forms(value, url)
else:
Log.error(str(parsed) + ' is not a valid parsed content!')
return forms
def my_ip(ip):
if APP_DEBUG:
Log.info('CALLED: Set.my_ip(' + str(ip) + ')')
if not validators.is_ip(ip):
Log.error(str(ip) + ' is not a valid ip address')
return False
return Set.__set__(keys.MY_IP, ip)
def team_player(ip='*'):
if APP_DEBUG:
Log.info('CALLED: Remove.team_player(' + str(ip) + ')')
if ip != '*' and not validators.is_ip(ip):
Log.error(str(ip) + ' is not a valid ip address')
return False
return Remove.__remove__(keys.TEAM_PLAYER, ip)
def submit_url(url):
if APP_DEBUG:
Log.info('CALLED: Set.submit_url(' + str(url) + ')')
if not validators.is_url(url):
Log.error(str(url) + ' is not a valid url')
return False
return Set.__set__(keys.SUBMIT_URL, url)
def game_server(ip):
if APP_DEBUG:
Log.info('CALLED: Set.game_server(' + str(ip) + ')')
if not validators.is_ip(ip):
Log.error(str(ip) + ' is not a valid ip address')
return False
return Set.__set__(keys.GAME_SERVER, ip)
def task_target(*arguments):
result = None
if self.tasks_type == MultiTask.MULTI_PROCESSING:
curr_task = multiprocessing.process.current_process()
Log.info(self.tag + 'started (PID=' + str(curr_task.pid) + ')')
else:
curr_task = threading.current_thread()
Log.info(self.tag + 'started')
if target is not None:
result = target(*arguments)
if result is not None:
Log.success("Result: " + str(result))
# Scrivo il risultato nel file
Log.info('Writing result in ' + str(self.resfile))
storage.overwrite_file(str(result), self.resfile) # TODO: dump result as object with "pickle"
# Termino tutti gli altri threads/processi
if self.tasks_type == MultiTask.MULTI_PROCESSING:
Log.info('Killing other processes')
running_pids = MultiTask.get_pids_from_file(self.pidfile)
for pid in running_pids:
pid = int(pid)
if pid == curr_task.pid:
continue
try:
os.kill(pid, signal.SIGKILL)
Log.info('Process ' + str(pid) + ' killed!')
except Exception as e:
Log.error(str(e))
Log.info(self.tag + 'end')
else:
Log.info('Ignoring other threads')
# Killa se stesso
pid = multiprocessing.process.current_process().pid
Log.info(self.tag + 'end')
os.kill(pid, signal.SIGKILL)
def get_dictionary(file: str) -> dict:
"""
:param file: A file that contains a json
:return: A dictionary
"""
if not os.path.isfile(file):
Log.error(file + ' is not a file')
return dict()
return JsonSerializer.load_json(file)
def _post_job(self, request) -> JsonResponse:
"""
:type request: django.core.handlers.wsgi.WSGIRequest
:return: django.http.JsonResponse
"""
# noinspection PyTypeChecker
job: AbstractJobModel = None
request_params: dict = request.POST.dict()
job_id = request_params.get('id')
try:
job_id = int(job_id)
job = self.model_class.objects.get(id=job_id)
except ValueError:
pass
except Exception as e:
Log.error(str(e))
if job is None:
return JsonResponse(
{'message': 'Unable to find the requested job'}, status=400)
signal_job = request_params.get('signal')
if signal_job is not None:
signal_job = int(signal_job)
if signal_job == 0: # Custom signal 0 = Restart capturing
job_new = self._copy_job(job)
job_id = job_new.id
signal_job = signal.SIGABRT
try:
job.kill(signal_job)
except ProcessLookupError:
Log.warning("The process " + str(job.pid) + " does not exists")
if signal_job == signal.SIGABRT: # 6 = Abort permanently by cleaning job
if not job.delete():
return JsonResponse(
{'message': 'Unable to delete the job'}, status=400)
return JsonResponse(
{
'id': job_id,
'signal': signal_job,
'message': 'Signal sent'
},
status=200)
job.self_check()
page = request_params.get('page')
page_size = request_params.get('page_size')
pagination = self.pagination(job.json_dict, page, page_size)
pagination.update({'job': {'id': job_id, 'status': job.status_name}})
return JsonResponse(pagination, status=200)
def find_forms(parsed: dict or list, url=None) -> dict:
"""
Search forms inside parsed html (dict)
:param parsed: A parsed html
:param url: The parsed url (or None)
:return: The list of found forms
"""
form = dict()
if parsed is None:
return form
if type(parsed) == dict:
parsed_children = parsed.get('children')
if 'form' == parsed.get('tag'):
attrs = parsed.get('attrs')
action = attrs.get('action')
method = attrs.get('method')
name = attrs.get('name')
if action is None:
action = url
if method is None:
method = HttpRequest.Type.POST
attrs = {'method': method, 'action': action, 'name': name}
if name is None:
name = action
form = {
'tag': 'form',
'attrs': attrs,
'name': name,
'children': HtmlParser.__find_inputs(parsed_children)
}
children = HtmlParser.find_forms(parsed_children, url)
if children is not None and len(children) > 0:
if len(form) > 0:
form['children'] = children
else:
if len(children) == 1 and children.get('tag') is None:
form = children.get(0)
else:
form = children
elif type(parsed) is list:
children = dict()
index = 0
for value in parsed:
child = HtmlParser.find_forms(value, url)
if len(child) > 0:
children[index] = child
index += 1
if len(children) == 1 and children.get(0).get('tag') is None:
form = children.get(0)
else:
form = children
else:
Log.error(str(parsed) + ' is not a valid parsed content!')
return form
def load_json(file: str) -> dict:
"""
:param file: The file to read
:return: A dictionary
"""
try:
with open(file, 'r') as infile:
return json.load(infile)
except json.decoder.JSONDecodeError as e:
Log.error(str(e))
return dict()
def decrypt(text: str):
for api in Md5Crypto.Api.all():
r = HttpRequest.request(api['url'] + text)
if r is None:
continue
try:
r_json = r.json()
except json.decoder.JSONDecodeError:
continue
result = api['get_result'](r_json)
if result is not None:
return result
Log.error('md5: unable to decrypt: ' + text)
return None
def request(url: str,
request_type: str = HttpRequest.Type.GET,
json: dict or list = None) -> dict:
"""
Send a request to sqlmap-api server and then load the data json as dict
:param url: The sqlmap-api url (eg. "http://127.0.0.1:8775/task/new")
:param request_type: get|post|put|patch|delete
:param json: The json to send
:rtype: dict
"""
response = HttpRequest.request(url, request_type, json=json)
r_data = JsonSerializer.load_json(response.text)
Log.info('Response data of ' + url + ': ' + str(r_data))
if not r_data['success']:
Log.error('Response data of ' + url + ' has { success: False }')
raise requests.RequestException('Request to ' + url + ' failed')
return r_data
def __init__(self, tasks_type):
if tasks_type not in MultiTask.tasks_types:
tasks_type = MultiTask.MULTI_PROCESSING
Log.error(str(tasks_type) + ' is not a valid tasks type!')
self.tasks_type = tasks_type
if self.tasks_type == MultiTask.MULTI_PROCESSING:
self.Multitask = multiprocessing.Process
self.tag = 'Process '
else:
self.Multitask = threading.Thread
self.tag = 'Thread '
self.tasks = []
pid = str(multiprocessing.process.current_process().pid)
# File con pids (se multiprocessing)
self.pidfile = APP_TMP + '/multitask.' + pid + '.' + timestamp() + '.pids'
# File con result
self.resfile = APP_TMP + '/multitask.' + pid + '.' + timestamp() + '.res'
def get(self, request, *args, **kwargs):
"""
:type request: django.core.handlers.wsgi.WSGIRequest
:return: django.http.HttpResponse
"""
request_params: dict = request.GET.dict()
try:
job_id = int(request_params.get('id'))
except (ValueError, TypeError) as e:
Log.error(str(e))
return redirect('/web/parsing')
job = self._get_job_model(job_id)
if job is None:
return redirect('/web/parsing/parse?id=' + str(job_id))
return FileResponse(open(job.json_file, 'rb'), as_attachment=True)
def request(url: str,
request_type: str = Type.GET,
data=None,
json: dict or list = None,
headers: dict = None) -> requests.Response or None:
"""
Make a request to chosen url
:param url: The target url
:param request_type: get|post|put|patch|delete
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`
:param json: (optional) json data to send in the body of the :class:`Request`
:param headers: The headers to send
:return The response of request, or None (if the request fail)
"""
if headers is None:
headers = {}
req_headers = {'User-Agent': str(APP_NAME) + ' ' + str(APP_VERSION)}
req_headers.update(headers)
if data is None:
data = {}
request_type = request_type.lower()
if not is_url(url):
Log.error(str(url) + ' is not a valid url!')
return None
try:
if request_type == HttpRequest.Type.GET:
response = requests.get(url, data, headers=req_headers)
elif request_type == HttpRequest.Type.POST:
response = requests.post(url, data, json, headers=req_headers)
elif request_type == HttpRequest.Type.PUT:
response = requests.put(url, data, headers=req_headers)
elif request_type == HttpRequest.Type.PATCH:
response = requests.patch(url, data, headers=req_headers)
elif request_type == HttpRequest.Type.DELETE:
response = requests.delete(url, headers=req_headers)
else:
Log.error(str(request_type) + ' is not a valid request type!')
return None
if APP_DEBUG:
HttpRequest.print_response(response)
return response
except (requests.exceptions.ConnectionError,
requests.exceptions.TooManyRedirects) as e:
Log.error('Unable to connect to ' + str(url))
Log.error('Exception: ' + str(e))
return None
def _get_job(self, request, redirect_url: str):
"""
Show the requested job
:type request: django.core.handlers.wsgi.WSGIRequest
:param redirect_url: The url to redirect the request in case of errors
:return: django.http.HttpResponse
"""
request_params: dict = request.GET.dict()
try:
job_id = int(request_params.get('id'))
except (ValueError, TypeError) as e:
Log.error(str(e))
return redirect(redirect_url)
Log.info("Showing job #" + str(job_id))
try:
job = self.model_class.objects.get(id=job_id)
except ObjectDoesNotExist:
return redirect(redirect_url)
return render(request, self.template_name, {'job': job})
def find_inputs(parsed: dict or list) -> dict:
"""
Search inputs inside a parsed html (dict)
:param parsed: A parsed html
:return: A dictionary of input tags like {'input[name]': {'attr1': 'attr1_val' ...}}
"""
inputs = {}
if parsed is None:
return inputs
if type(parsed) == dict:
tag = parsed.get('tag')
if tag in ('input', 'textarea'):
attrs = parsed.get('attrs')
form_input = {'tag': tag}
for key, value in attrs.items():
form_input[key] = value
inputs[attrs.get('name')] = form_input
inputs.update(HtmlParser.find_inputs(parsed.get('children')))
elif type(parsed) == list:
for value in parsed:
inputs.update(HtmlParser.find_inputs(value))
else:
Log.error(str(parsed) + ' is not a valid parsed content!')
return inputs
def _update_manufacturer_dict(self):
manufacturer_response = HttpRequest.request(
MacManufacturer._MANUFACTURERS_URL)
if manufacturer_response is None:
return
if manufacturer_response.text is None:
return
manufacturer_dict = dict()
manufacturer_list = manufacturer_response.text.splitlines()
for manufacturer in manufacturer_list:
if len(manufacturer) < 1:
continue
if manufacturer[0] == '#':
continue
manufacturer_details = manufacturer.split('\t')
i = 0
mac = None
lookup_dict = {
MacManufacturer._MANUFACTURERS_DETAIL_DICT[1]: None,
MacManufacturer._MANUFACTURERS_DETAIL_DICT[2]: None,
MacManufacturer._MANUFACTURERS_DETAIL_DICT[3]: None
}
for detail in manufacturer_details:
if detail == '':
continue
if i == 0:
# MAC address
mac_detail = detail.split('/')
if len(mac_detail) == 2:
# The mac has a sub mask, so the dict key is the first n bits
sub_mask = int(mac_detail[1]) / 4
mac_sub_mask = floor(sub_mask + (sub_mask / 2))
mac = mac_detail[0][0:mac_sub_mask]
elif len(mac_detail) == 1:
# The mac has not a sub mask
mac = mac_detail[0]
else:
Log.error("Wrong mac address: " + str(detail))
break
if i >= len(MacManufacturer._MANUFACTURERS_DETAIL_DICT):
Log.error("Wrong manufacturer details: " +
str(manufacturer_details))
break
lookup_dict[
MacManufacturer._MANUFACTURERS_DETAIL_DICT[i]] = detail
i += 1
if mac is None:
Log.error("Wrong manufacturer details: " +
str(manufacturer_details))
continue
manufacturer_dict[mac] = lookup_dict
if len(manufacturer_dict) > 0:
self._manufacturer_dict = manufacturer_dict
JsonSerializer.set_dictionary(self._manufacturer_dict,
MacManufacturer._MANUFACTURERS_JSON)
def __find_tags(parsed: dict or list, tags: dict) -> dict:
"""
Search a certain kinds of tag inside a parsed html (dict)
:param parsed: A parsed html
:tags parsed: The tags to find
:return: A dictionary of tags like {'tag[name]': {'attr1': 'attr1_val' ...}}
"""
found_tag = {}
if parsed is None:
return found_tag
if type(parsed) is dict:
tag: str = parsed.get('tag')
if tag is not None:
tag = tag.lower()
if tag in tags:
attrs = parsed.get('attrs')
# Tag
found_tag = {'tag': tag}
# Tag attrs
found_tag_attrs = dict()
for key, value in attrs.items():
found_tag_attrs[key] = value.strip()
found_tag['attrs'] = found_tag_attrs
# Tag data
found_tag['data'] = parsed.get('data')
# Tag name
name = None
for tag_name in HtmlParser._tag_names:
name = attrs.get(tag_name)
if name is not None:
break
if name is None:
for tag_key in HtmlParser._tag_key_names:
name = attrs.get(tag_key)
if name is not None:
name = tag_key
break
if name is None:
name = tag
found_tag['name'] = name
# Tag children
children = HtmlParser.__find_tags(parsed.get('children'), tags)
if children is not None and len(children) > 0:
if len(found_tag) > 0:
found_tag['children'] = children
else:
if len(children
) == 1 and children.get(0).get('name') is None:
found_tag = children.get(0)
else:
found_tag = children
elif type(parsed) is list:
children = dict()
index = 0
for value in parsed:
child = HtmlParser.__find_tags(value, tags)
if len(child) > 0:
children[index] = child
index += 1
if len(children) == 1 and children.get(0).get('name') is None:
found_tag = children.get(0)
else:
found_tag = children
else:
Log.error(str(parsed) + ' is not a valid parsed content!')
return found_tag
def request(url: str,
request_type: str = Type.GET,
data=None,
json: dict or list = None,
headers: dict = None,
timeout: int = DEFAULT_TIMEOUT,
cookies: str or dict = None) -> requests.Response or None:
"""
Make a request to chosen url
:param url: The target url
:param request_type: get|post|put|patch|delete
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`
:param json: (optional) json data to send in the body of the :class:`Request`
:param headers: The headers to send
:param timeout: The request timeout
:param cookies: The request cookies
:return The response of request, or None (if the request fail)
"""
if headers is None:
headers = {}
if type(cookies) is str:
try:
cookies = dict(
(k.strip(), v.strip())
for k, v in (c.split('=') for c in cookies.split(';')))
except ValueError:
# Wrong or empty cookies
cookies = None
pass
req_headers = {'User-Agent': str(APP_NAME) + ' ' + str(APP_VERSION)}
req_headers.update(headers)
if data is None:
data = {}
request_type = request_type.lower()
if not is_url(url):
Log.error(str(url) + ' is not a valid url!')
return None
try:
if request_type == HttpRequest.Type.GET:
response = requests.get(url,
data,
headers=req_headers,
timeout=timeout,
cookies=cookies)
elif request_type == HttpRequest.Type.POST:
response = requests.post(url,
data,
json,
headers=req_headers,
timeout=timeout,
cookies=cookies)
elif request_type == HttpRequest.Type.PUT:
response = requests.put(url,
data,
headers=req_headers,
timeout=timeout,
cookies=cookies)
elif request_type == HttpRequest.Type.PATCH:
response = requests.patch(url,
data,
headers=req_headers,
timeout=timeout,
cookies=cookies)
elif request_type == HttpRequest.Type.DELETE:
response = requests.delete(url,
headers=req_headers,
timeout=timeout,
cookies=cookies)
else:
Log.error(str(request_type) + ' is not a valid request type!')
return None
if APP_DEBUG:
HttpRequest.print_response(response)
return response
except (requests.exceptions.ConnectionError,
requests.exceptions.TooManyRedirects,
requests.exceptions.ReadTimeout) as e:
Log.error('Unable to complete request to ' + str(url))
Log.error('Exception: ' + str(e))
return None
def start(self, target, args, asynchronous, cpu):
self.tasks = []
def task_target(*arguments):
result = None
if self.tasks_type == MultiTask.MULTI_PROCESSING:
curr_task = multiprocessing.process.current_process()
Log.info(self.tag + 'started (PID=' + str(curr_task.pid) + ')')
else:
curr_task = threading.current_thread()
Log.info(self.tag + 'started')
if target is not None:
result = target(*arguments)
if result is not None:
Log.success("Result: " + str(result))
# Scrivo il risultato nel file
Log.info('Writing result in ' + str(self.resfile))
storage.overwrite_file(
str(result),
self.resfile) # TODO: dump result as object with "pickle"
# Termino tutti gli altri threads/processi
if self.tasks_type == MultiTask.MULTI_PROCESSING:
Log.info('Killing other processes')
running_pids = MultiTask.get_pids_from_file(self.pidfile)
for pid in running_pids:
pid = int(pid)
if pid == curr_task.pid:
continue
try:
os.kill(pid, signal.SIGKILL)
Log.info('Process ' + str(pid) + ' killed!')
except Exception as ex:
Log.error(str(ex))
Log.info(self.tag + 'end')
else:
Log.info('Ignoring other threads')
# Killa se stesso
pid = multiprocessing.process.current_process().pid
Log.info(self.tag + 'end')
os.kill(pid, signal.SIGKILL)
for i in range(0, cpu):
task_args = ()
for arg in args:
Log.info('Argument type: ' + str(type(arg)))
if is_listable(arg):
# Divido gli elementi in 1/cpu parti
p_list_len = (len(arg) / cpu) + (len(arg) % cpu)
if type(arg) == dict:
iterator = iter(arg.items())
task_args += (dict(
itertools.islice(iterator, int((i * p_list_len)),
int((i + 1) * p_list_len))), )
else:
task_args += (arg[int((
i * p_list_len)):int(((i + 1) * p_list_len))], )
else:
task_args += (arg, )
task = self.Multitask(target=task_target, args=task_args)
self.tasks.append(task)
if self.tasks_type == MultiTask.MULTI_PROCESSING:
pids = []
try:
signal.signal(signal.SIGCHLD,
signal.SIG_IGN) # Ignore child exit status
except ValueError as e:
# Probably you have executed Django
Log.error(str(e))
for task in self.tasks:
task.start()
# noinspection PyUnresolvedReferences
pids.append(task.pid)
storage.overwrite_file(str(pids).strip('[]'), self.pidfile)
else:
for task in self.tasks:
task.start()
if not asynchronous:
# Attende la fine dell'esecuzione di tutti i tasks
for task in self.tasks:
task.join()
Log.info('Task ' + str(task.name) + ' joined')
Log.info('Reading result in ' + str(self.resfile))
# Prendo il risultato dal file
res = storage.read_file(
self.resfile) # TODO: load result as object with "pickle"
# Elimino l'eventuale file con i pid
storage.delete(self.pidfile)
# Elimino il file con il risultato
storage.delete(self.resfile)
Log.success('MultiTask -> result: ' + str(res))
return res
return None