def _parse(self, url: str = None, html: str = None) -> (dict, str):
"""
Make an HTML/URL parsing by processing ALL found tags
:param url: The url to parse (or None)
:param html: The html page to parse as string (or None)
:return: dictionary of tags, cookies
"""
self.url = None
self.base_url = None
cookies = ''
if url is not None:
self.url = url
url_parsed = urlparse(url)
self.url_scheme = str(url_parsed.scheme)
self.base_url = self.url_scheme + '://' + str(url_parsed.netloc)
r = HttpRequest.request(url)
if r is None:
return None
try:
html = r.json()
Log.warning('Trying to parse a json with HTML parser!')
except ValueError:
html = r.text
if r.headers is not None:
for k in r.headers.keys():
if k == 'Set-Cookie' or k == 'set-cookie' or k == 'Set-cookie':
cookies = r.headers.get('Set-Cookie')
break
sorted_html, errors = tidy_document(html) # Sort html (and fix errors)
self.feed(sorted_html)
return self.tags, cookies
def _post_job(self, request) -> JsonResponse:
"""
:type request: django.core.handlers.wsgi.WSGIRequest
:return: django.http.JsonResponse
"""
# noinspection PyTypeChecker
job: AbstractJobModel = None
request_params: dict = request.POST.dict()
job_id = request_params.get('id')
try:
job_id = int(job_id)
job = self.model_class.objects.get(id=job_id)
except ValueError:
pass
except Exception as e:
Log.error(str(e))
if job is None:
return JsonResponse(
{'message': 'Unable to find the requested job'}, status=400)
signal_job = request_params.get('signal')
if signal_job is not None:
signal_job = int(signal_job)
if signal_job == 0: # Custom signal 0 = Restart capturing
job_new = self._copy_job(job)
job_id = job_new.id
signal_job = signal.SIGABRT
try:
job.kill(signal_job)
except ProcessLookupError:
Log.warning("The process " + str(job.pid) + " does not exists")
if signal_job == signal.SIGABRT: # 6 = Abort permanently by cleaning job
if not job.delete():
return JsonResponse(
{'message': 'Unable to delete the job'}, status=400)
return JsonResponse(
{
'id': job_id,
'signal': signal_job,
'message': 'Signal sent'
},
status=200)
job.self_check()
page = request_params.get('page')
page_size = request_params.get('page_size')
pagination = self.pagination(job.json_dict, page, page_size)
pagination.update({'job': {'id': job_id, 'status': job.status_name}})
return JsonResponse(pagination, status=200)
def __parse(self,
url: str = None,
html: str = None,
cookies: str = None) -> (dict, str):
"""
Make an HTML/URL parsing by processing ALL found tags
:param url: The url to parse (or None)
:param html: The html page to parse as string (or None)
:param cookies: The cookies to use on parsing
:return: dictionary of tags, cookies
"""
self.url = None
self.base_url = None
is_image = False
if url is not None:
self.url = url
url_parsed = urlparse(url)
self.url_scheme = str(url_parsed.scheme)
self.base_url = self.url_scheme + '://' + str(url_parsed.netloc)
r = HttpRequest.request(url, cookies=cookies)
if r is None:
return None
if r.status_code >= 400 or r.headers.get(
'Content-Type') in HtmlParser._unacceptable_content_types:
return None
try:
html = r.json()
Log.warning('Trying to parse a json with HTML parser!')
except ValueError:
html = r.text
if r.headers is not None:
for k, v in r.headers.items():
if k.lower() == 'set-cookie':
cookies = v
if HttpRequest.is_image(r):
is_image = True
xmp_start = html.find('<x:xmpmeta')
xmp_end = html.find('</x:xmpmeta')
xmp_str = html[xmp_start:xmp_end + 12]
html = xmp_str
if is_image:
sorted_html = html
else:
sorted_html, errors = tidy_document(
html) # Sort html (and fix errors)
self.feed(sorted_html)
if cookies is None:
cookies = ''
return self.tags, cookies
def post(self, request):
"""
:type request: django.core.handlers.wsgi.WSGIRequest
:return: django.http.HttpResponse
"""
if not util.is_root():
return JsonResponse({'message': 'You are not #root'},
status=401)
# noinspection PyTypeChecker
sniffing_job: SniffingJobModel = None
request_params: dict = request.POST.dict()
sniffing_job_id = request_params.get('id')
try:
sniffing_job_id = int(sniffing_job_id)
sniffing_job = SniffingJobModel.objects.get(id=sniffing_job_id)
except ValueError:
pass
except Exception as e:
print(type(e))
print(str(e))
if sniffing_job is None:
return JsonResponse(
{'message': 'Unable to find the requested job'},
status=400)
signal_job = request_params.get('signal')
if signal_job is not None:
signal_job = int(signal_job)
if signal_job == 0: # Custom signal 0 = Restart capturing
sniffing_job_new = self.new_job(
sniffing_job.filters,
sniffing_job.pcap_file,
sniffing_job.interfaces,
)
sniffing_job_id = sniffing_job_new.id
signal_job = signal.SIGABRT
try:
sniffing_job.kill(signal_job)
except ProcessLookupError:
Log.warning("The process " + str(sniffing_job.pid) +
" does not exists")
if signal_job == signal.SIGABRT: # 6 = Abort permanently by cleaning job
if not sniffing_job.delete():
return JsonResponse(
{'message': 'Unable to delete the job'})
return JsonResponse(
{
'id': sniffing_job_id,
'signal': signal_job,
'message': 'Signal sent'
},
status=200)
page = request_params.get('page')
page_size = request_params.get('page_size')
pagination = self.pagination(sniffing_job.json_dict, page,
page_size)
pagination.update({
'job': {
'id': sniffing_job_id,
'status': sniffing_job.status_name
}
})
return JsonResponse(pagination, status=200)