def task_target(*arguments): result = None if self.tasks_type == MultiTask.MULTI_PROCESSING: curr_task = multiprocessing.process.current_process() Log.info(self.tag + 'started (PID=' + str(curr_task.pid) + ')') else: curr_task = threading.current_thread() Log.info(self.tag + 'started') if target is not None: result = target(*arguments) if result is not None: Log.success("Result: " + str(result)) # Scrivo il risultato nel file Log.info('Writing result in ' + str(self.resfile)) storage.overwrite_file(str(result), self.resfile) # TODO: dump result as object with "pickle" # Termino tutti gli altri threads/processi if self.tasks_type == MultiTask.MULTI_PROCESSING: Log.info('Killing other processes') running_pids = MultiTask.get_pids_from_file(self.pidfile) for pid in running_pids: pid = int(pid) if pid == curr_task.pid: continue try: os.kill(pid, signal.SIGKILL) Log.info('Process ' + str(pid) + ' killed!') except Exception as e: Log.error(str(e)) Log.info(self.tag + 'end') else: Log.info('Ignoring other threads') # Killa se stesso pid = multiprocessing.process.current_process().pid Log.info(self.tag + 'end') os.kill(pid, signal.SIGKILL)
def deep_inject_form(href, depth=0): # Check the domain if href in parsed_forms or \ urlparse(href).netloc != base_url or \ (max_depth is not None and depth > max_depth): return '' # Visit the current href parsed_relevant, request_cookies = HtmlParser.relevant_parse(href) # Find forms in page parsed_forms[href] = HtmlParser.find_forms(parsed_relevant, href) # Execute Sqlmap task task = SqlmapClient.try_inject_form(href, parsed_forms, request_cookies) tasks[task.id] = task Log.success('SQL injection of "' + href + '" started!') # Find adjacent links links = HtmlParser.find_links(parsed_relevant) # Visit adjacent links for link in links: # print('link: '+link) child_request_cookies = deep_inject_form(link, depth + 1) if len(child_request_cookies) > len(request_cookies): request_cookies = child_request_cookies
def browser_target(): """ The function that launch the browser """ set_owner_process(user) Log.info('Launching browser with User: '******'Web browser opened')
def kill(self, sig: int): """ Send a signal to process which is running this job :param sig: The signal as integer (eg. 9 for SIGKILL) """ Log.info("Sending signal " + str(sig) + " to job #" + str(self.id) + ' (' + str(self.pid) + ')') self.status = sig os.kill(self.pid, sig) self.save() Log.success("Signal " + str(sig) + " sent to job #" + str(self.id) + ' (' + str(self.pid) + ')')
def deep_inject_form(url, max_depth, listen: bool = False) -> dict: """ Search a form in the page returned by url. If it doesn't find a form, or the injection can't be done, it visit the website in search for other forms :param listen: True if this method should listen and print the SQL tasks, otherwise False :param url: str The url to visit :param max_depth: int The max depth during the visit :return A dictionary of SQL injection tasks """ base_url = urlparse(url).netloc parsed_forms = dict() out_file = APP_STORAGE_OUT + '/' + now( ) + '_DEEP_FORMS_' + base_url + '.json' def _deep_inject_form(href, depth=1): # Check the domain if href in parsed_forms or \ urlparse(href).netloc != base_url or \ (max_depth is not None and depth > max_depth): return '' # Visit the current href parsed_relevant, request_cookies = HtmlParser.relevant_parse(href) # Find forms in page parsed_forms[href] = HtmlParser.find_forms(parsed_relevant, href) # Find adjacent links links = HtmlParser.find_links(parsed_relevant) if len(parsed_forms) % 10 == 0: Log.info('Writing result in ' + out_file + '...') JsonSerializer.set_dictionary(parsed_forms, out_file) # Visit adjacent links for link in links: # print('link: '+link) child_request_cookies = _deep_inject_form(link, depth + 1) if len(child_request_cookies) > len(request_cookies): request_cookies = child_request_cookies return request_cookies cookies = _deep_inject_form(url) Log.info('Writing result in ' + out_file + '...') JsonSerializer.set_dictionary(parsed_forms, out_file) Log.success('Result wrote in ' + out_file) Log.success('Website crawled! Found ' + str(len(parsed_forms)) + ' pages') tasks = SqlmapClient.try_inject_forms(parsed_forms, cookies) if listen: SqlInjection.__listen_tasks(tasks) return tasks
def inject_form(url=None, html=None): """ Search a form in the page returned by url (or inside the html). :param url: str The url to visit (or None) :param html: str the html code to analyze (or None) :return A list of parsed forms like [ form_1, form_2 ] """ parsed_forms = dict() parsed_forms[url], cookies = HtmlParser.form_parse(url, html) Log.success('Html parsed! Found ' + str(len(parsed_forms[url])) + ' forms') SqlmapClient.try_inject(parsed_forms, cookies)
def __init__(self, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT): """ :param host: The host :param port: The port """ self.host = host self.port = port self.base_url = 'http://' + self.host + ':' + str(port) # Start the sqlmap-api server in a parallel thread Log.info("Starting sqlmap-api server in a parallel thread") MultiTask.multithread(sqlmap_server, (self.host, self.port), True, 1) while not check_socket(self.host, self.port): # Wait sqlmap-api server sleep(0.1) Log.success("Sqlmap-api server started!")
def __inject_forms(url, max_depth) -> dict: """ Search a form in the page returned by url. If it doesn't find a form, or the injection can't be done, it visit the website in search for other forms :param url: str The url to visit :param max_depth: int The max depth during the visit :return A dictionary of SQL injection tasks """ base_url = urlparse(url).netloc parsed_forms = dict() tasks = dict() def deep_inject_form(href, depth=0): # Check the domain if href in parsed_forms or \ urlparse(href).netloc != base_url or \ (max_depth is not None and depth > max_depth): return '' # Visit the current href parsed_relevant, request_cookies = HtmlParser.relevant_parse(href) # Find forms in page parsed_forms[href] = HtmlParser.find_forms(parsed_relevant, href) # Execute Sqlmap task task = SqlmapClient.try_inject_form(href, parsed_forms, request_cookies) tasks[task.id] = task Log.success('SQL injection of "' + href + '" started!') # Find adjacent links links = HtmlParser.find_links(parsed_relevant) # Visit adjacent links for link in links: # print('link: '+link) child_request_cookies = deep_inject_form(link, depth + 1) if len(child_request_cookies) > len(request_cookies): request_cookies = child_request_cookies Log.success('SQL injection started!') deep_inject_form(url) Log.success('Website crawled! Found ' + str(len(parsed_forms)) + ' forms') return tasks
def kill(self, sig: int): """ Send a signal to process which is running this job :param sig: The signal as integer (eg. 9 for SIGKILL) """ if self.status == sig: return Log.info("Sending signal " + str(sig) + " to job #" + str(self.id) + ' (' + str(self.pid) + ')') self.status = sig try: os.kill(self.pid, sig) except ProcessLookupError: # Process does not exists pass self.save() Log.success("Signal " + str(sig) + " sent to job #" + str(self.id) + ' (' + str(self.pid) + ')')
def inject_form(url: str = None, html: str = None, listen: bool = False) -> dict: """ Search a form in the page returned by url (or inside the html). :param listen: True if this method should listen and print the SQL tasks, otherwise False :param url: str The url to visit (or None) :param html: str the html code to analyze (or None) :return A dictionary of SQL injection tasks """ parsed_forms = dict() parsed_forms[url], cookies = HtmlParser.form_parse(url, html) Log.success('Html parsed! Found ' + str(len(parsed_forms[url])) + ' forms') tasks = SqlmapClient.try_inject_forms(parsed_forms, cookies) if listen: SqlInjection.__listen_tasks(tasks) return tasks
def start(self, target, args, asynchronous, cpu): self.tasks = [] def task_target(*arguments): result = None if self.tasks_type == MultiTask.MULTI_PROCESSING: curr_task = multiprocessing.process.current_process() Log.info(self.tag + 'started (PID=' + str(curr_task.pid) + ')') else: curr_task = threading.current_thread() Log.info(self.tag + 'started') if target is not None: result = target(*arguments) if result is not None: Log.success("Result: " + str(result)) # Scrivo il risultato nel file Log.info('Writing result in ' + str(self.resfile)) storage.overwrite_file(str(result), self.resfile) # TODO: dump result as object with "pickle" # Termino tutti gli altri threads/processi if self.tasks_type == MultiTask.MULTI_PROCESSING: Log.info('Killing other processes') running_pids = MultiTask.get_pids_from_file(self.pidfile) for pid in running_pids: pid = int(pid) if pid == curr_task.pid: continue try: os.kill(pid, signal.SIGKILL) Log.info('Process ' + str(pid) + ' killed!') except Exception as e: Log.error(str(e)) Log.info(self.tag + 'end') else: Log.info('Ignoring other threads') # Killa se stesso pid = multiprocessing.process.current_process().pid Log.info(self.tag + 'end') os.kill(pid, signal.SIGKILL) for i in range(0, cpu): task_args = () for arg in args: Log.info('Argument type: ' + str(type(arg))) if is_listable(arg): # Divido gli elementi in 1/cpu parti p_list_len = (len(arg) / cpu) + (len(arg) % cpu) if type(arg) == dict: iterator = iter(arg.items()) task_args += ( dict(itertools.islice(iterator, int((i * p_list_len)), int((i + 1) * p_list_len))), ) else: task_args += (arg[int((i * p_list_len)):int(((i + 1) * p_list_len))],) else: task_args += (arg,) task = self.Multitask(target=task_target, args=task_args) self.tasks.append(task) if self.tasks_type == MultiTask.MULTI_PROCESSING: pids = [] signal.signal(signal.SIGCHLD, signal.SIG_IGN) # Ignore child exit status for task in self.tasks: task.start() # noinspection PyUnresolvedReferences pids.append(task.pid) storage.overwrite_file(str(pids).strip('[]'), self.pidfile) else: for task in self.tasks: task.start() if not asynchronous: # Attende la fine dell'esecuzione di tutti i tasks for task in self.tasks: task.join() Log.info('Task ' + str(task.name) + ' joined') Log.info('Reading result in ' + str(self.resfile)) # Prendo il risultato dal file res = storage.read_file(self.resfile) # TODO: load result as object with "pickle" # Elimino l'eventuale file con i pid storage.delete(self.pidfile) # Elimino il file con il risultato storage.delete(self.resfile) Log.success('MultiTask -> result: ' + str(res)) return res return None
def crawl(url: str, parsing_type: str, callback, depth: int = 0, cookies: str = None): """ :param url: The url to crawl/parse :param parsing_type: HtmlParse.TYPE_ALL | HtmlParse.TYPE_RELEVANT | HtmlParse.TYPE_FORM | HtmlParse.TYPE_META :param callback: The callback method to call foreach visited page :param depth: The max crawling depth (0 to execute a normal page parsing, < 0 for no limit) :param cookies: The cookies to use on parsing """ if not is_url(url): raise ValueError('url must be a valid url') if parsing_type not in HtmlParser.types(): raise ValueError('parsing_type must be one of ' + str(HtmlParser.types())) if not callable(callback): raise ValueError('callback is not callable') if type(depth) is not int: raise ValueError('dept must be an integer') if cookies is not None and type(cookies) is not str: raise ValueError('cookies must be a string') base_url = urlparse(url).netloc base_urls = (base_url, ) if base_url[0:4] != 'www.': base_urls += ('www.' + str(base_url), ) parsed_urls = set() parsed_hashes = set() def _crawl(href: str, curr_depth: int = 0): if href in parsed_urls or \ urlparse(href).netloc not in base_urls or \ (0 <= depth and (depth < curr_depth)): return # Visit the current href if parsing_type == HtmlParser.TYPE_ALL: parsed, _ = HtmlParser.all_parse(href, cookies=cookies) else: parsed, _ = HtmlParser.relevant_parse(href, cookies=cookies) parsed_hash = hash(JsonSerializer.dump_json(parsed)) if parsed_hash in parsed_hashes: return parsed_hashes.add(parsed_hash) parsed_urls.add(href) if parsing_type == HtmlParser.TYPE_FORM: # Find forms in page parsed_page = HtmlParser.find_forms(parsed, href) elif parsing_type == HtmlParser.TYPE_META: # Find metadata in page parsed_page = HtmlParser.find_meta(parsed) else: parsed_page = parsed if parsed_page.get('tag') is not None: parsed_page = {0: parsed_page} parsed_page['url'] = href callback(parsed_page) # Find adjacent links links = HtmlParser.find_links(parsed) for link in links: _crawl(link, curr_depth + 1) _crawl(url) Log.success(url + ' crawling done!')