def get_a_task(self): try: task = self.queue.get(block=True, timeout=self.timeout) self.process(task) self.queue.task_done() Console.body(task) return True except Queue.Empty: return False
def get_a_task(self, alived): try: task = self.queue.get(False) if task: self.process(task) self.queue.task_done() Console.body(task) else: print("no task") return True except Queue.Empty: return bool(alived)
def get_a_task(self, alived): try: task = self.queue.get(False) if task: self.process(task) self.queue.task_done() Console.body(task) else: print "no task" return True except Queue.Empty: if not alived: return False else: return True
# # Create the thread_pool and start the daemonized threads # thread_pool = [] for visitor_id in range(0, threads): v = Visitor(visitor_id, payload_queue, manager) thread_pool.append(v) v.daemon = True v.start() # # Run the main thread until manager exhaust all tasks # time_before_running = time.time() Console.header() while True: try: if not manager.get_a_task() and any([visitor.is_alive() for visitor in thread_pool]): print("Finishing...") break except KeyboardInterrupt: print (os.linesep + "Waiting for threads to stop...") Visitor.kill() break time_after_running = time.time() delta = round(timedelta(seconds=(time_after_running - time_before_running)).total_seconds(), 2) print("Task took %i seconds" % delta)
) sys.exit() # Select if full path is prefered full_path = args.full_path # Select if user wants content-type print show_content_type = args.show_content_type # # Run the main thread until manager exhaust all tasks # time_before_running = time.time() Console.number_of_requests = payload.get_total_requests() Console.number_of_threads = threads Console.start_eta_queue(30) Console.show_full_path = full_path Console.show_content_type = show_content_type Console.header() Console.set_show_progress(False if args.no_progress else True) Console.set_show_colors(False if args.no_colors else True) # # Create the thread_pool and start the daemonized threads # thread_pool = [] for visitor_id in range(threads): v = Visitor(visitor_id, payload_queue, manager, manager_lock) thread_pool.append(v) v.start()
(total_requests, int(total_requests / threads))) # # Create the thread_pool and start the daemonized threads # thread_pool = [] for visitor_id in range(0, threads): v = Visitor(visitor_id, payload_queue, manager) thread_pool.append(v) v.daemon = True v.start() # # Run the main thread until manager exhaust all tasks # time_before_running = time.time() Console.start_eta_queue(30, total_requests) Console.header() try: while True: visitors_alive = any([visitor.is_alive() for visitor in thread_pool]) if not manager.get_a_task(visitors_alive): break except KeyboardInterrupt: sys.stdout.write(os.linesep + "Waiting for threads to stop...") Visitor.kill() resp = raw_input(os.linesep + "Keep a resume file? [y/N] ") if resp == 'y': resumer.set_line(payload_queue.get().get_number()) with open("resume_file_" + time.strftime("%d_%m_%y_%H_%M", time.localtime()), 'w') as f: pickle.dump(resumer, f)
) sys.exit() # Select if full path is prefered full_path = args.full_path # Select if user wants content-type print show_content_type = args.show_content_type # # Run the main thread until manager exhaust all tasks # time_before_running = time.time() Console.number_of_requests = payload.get_total_requests() Console.number_of_threads = threads Console.start_eta_queue(30) Console.show_full_path = full_path Console.show_content_type = show_content_type Console.header() # # Create the thread_pool and start the daemonized threads # thread_pool = [] for visitor_id in range(threads): v = Visitor(visitor_id, payload_queue, manager, manager_lock) thread_pool.append(v) v.start() # Main loop try:
try: with open(resume, "rb") as f: resumer = pickle.load(f) args = resumer.get_args() except Exception: sys.stdout.write("[!] Could not load a correct resume file, sorry.") sys.exit() # Target check and preparation if not args.target: print("[!] You need to specify a target") parser.print_help() sys.exit() target = prepare_target(args.target) Console.add_config("{:37} {:>}".format("Target:", target)) recursive = args.recursive Console.add_config("{:37} {:>}".format("Recursive requests", "Yes" if recursive else "No")) # Persistent connections persist = args.persist Console.add_config("{:37} {:>}".format("Persistent connections", "Yes" if persist else "No")) # Misc options extension = args.extension.split(",") threads = int(args.threads) banned_response_codes = [int(item) for item in args.banned.split(",") if item] unbanned_response_codes = [
print("[!] Error setting cookies. Review cookie string (key:value,key:value...)") sys.exit() # Select if full path is prefered full_path = args.full_path # Select if user wants content-type print show_content_type = args.show_content_type # # Run the main thread until manager exhaust all tasks # time_before_running = time.time() Console.number_of_requests = payload.get_total_requests() Console.number_of_threads = threads Console.start_eta_queue(30) Console.show_full_path = full_path Console.show_content_type = show_content_type Console.header() # # Create the thread_pool and start the daemonized threads # thread_pool = [] for visitor_id in range(threads): v = Visitor(visitor_id, payload_queue, manager, manager_lock) thread_pool.append(v) v.start() # Main loop try:
# # Create the thread_pool and start the daemonized threads # thread_pool = [] for visitor_id in range(0, threads): v = Visitor(visitor_id, payload_queue, manager) thread_pool.append(v) v.daemon = True v.start() # # Run the main thread until manager exhaust all tasks # time_before_running = time.time() Console.header() try: while True: visitors_alive = any([v.is_alive() for v in thread_pool]) if not manager.get_a_task(visitors_alive): break except KeyboardInterrupt: sys.stdout.write(os.linesep + "Waiting for threads to stop...") Visitor.kill() resp = raw_input(os.linesep + "Resume file? (Type 'y' to get) ") if resp == 'y': resumer.set_line(payload_queue.get().get_number()) with open( "resume_file_" + time.strftime("%d_%m_%y_%H_%M", time.localtime()), 'w') as f:
def visit(self, task): def _dumb_redirect(url): origin = "{0}{1}".format(task.target, task.resource) # Detect redirect to same page but ended with slash if url == origin: return True if url == origin + '/': return True # Detect redirect to root if url == task.target: return True return False try: if Visitor.user_agent: Visitor.headers['User-Agent'] = Visitor.user_agent now = time.time() timeout = sum(self.__time) / len(self.__time) if self.__time else 10 # Persistent connections if Visitor.persist: if not self.session: self.session = requests.Session() else: self.session = requests r = None if Visitor.proxy: if Visitor.requests == "GET": r = self.session.get(task.get_complete_target(), headers=Visitor.headers, proxies=Visitor.proxy, verify=False, timeout=timeout, auth=Visitor.auth, cookies=Visitor.cookies, allow_redirects=Visitor.is_allow_redirects) elif Visitor.requests == "HEAD": r = self.session.head(task.get_complete_target(), headers=Visitor.headers, proxies=Visitor.proxy, verify=False, timeout=timeout, auth=Visitor.auth, cookies=Visitor.cookies, allow_redirects=Visitor.is_allow_redirects) else: if Visitor.requests == "GET": r = self.session.get(task.get_complete_target(), headers=Visitor.headers, verify=False, timeout=timeout, auth=Visitor.auth, cookies=Visitor.cookies, allow_redirects=Visitor.is_allow_redirects) elif Visitor.requests == "HEAD": r = self.session.head(task.get_complete_target(), headers=Visitor.headers, verify=False, timeout=timeout, auth=Visitor.auth, cookies=Visitor.cookies, allow_redirects=Visitor.is_allow_redirects) after = time.time() delta = (after - now) * 1000 tmp_content = r.content task.response_size = len(tmp_content) task.response_time = delta task.set_response_code(r.status_code) self.__time.append(delta) # If discriminator is found we mark it 404 if sys.version_info[0] >= 3: tmp_content = tmp_content.decode('Latin-1') if Visitor.discriminator and Visitor.discriminator in tmp_content: task.ignorable = True if Visitor.banned_md5 and hashlib.md5("".join(tmp_content)).hexdigest() == self.banned_md5: task.ignorable = True # Check if page size is not what we need if task.response_size in Visitor.size_discriminator: task.ignorable = True # Look for interesting content if task.content and (task.content in tmp_content): task.content_has_detected(True) # Look for a redirection if Visitor.is_allow_redirects: if len(r.history) > 0 and not _dumb_redirect(r.history[-1].url): task.response_code = str(r.history[0].status_code) task.location = r.history[-1].url else: if str(r.status_code).startswith('3'): task.set_response_code('404') task.ignorable = True if 'content-type' in [h.lower() for h in r.headers.keys()]: try: task.response_type = r.headers['Content-Type'].split(';')[0] except: pass self.lock.acquire() Console.body(task) self.results.get_results_queue().put(task) self.results.get_a_task() if Visitor.delay: time.sleep(Visitor.delay) except (requests.ConnectionError, requests.Timeout) as e: # TODO log to a file instead of screen print ("[!] Timeout/Connection error") print (e) except Exception as e: print ("[!] General exception while visiting") print (e) finally: self.lock.release()
def put(self, task): Console.update(task) self.queue.put(task)
def visit(self, task): def _dumb_redirect(url): origin = "{0}{1}".format(task.target, task.resource) # Detect redirect to same page but ended with slash if url == origin: return True if url == origin + '/': return True # Detect redirect to root if url == task.target: return True return False try: if Visitor.user_agent: Visitor.headers['User-Agent'] = Visitor.user_agent now = time.time() timeout = sum(self.__time) / len( self.__time) if self.__time else 10 # Persistent connections if Visitor.persist: if not self.session: self.session = requests.Session() else: self.session = requests r = None if Visitor.proxy: if Visitor.requests == "GET": r = self.session.get( task.get_complete_target(), headers=Visitor.headers, proxies=Visitor.proxy, verify=False, timeout=timeout, auth=Visitor.auth, cookies=Visitor.cookies, allow_redirects=Visitor.is_allow_redirects) elif Visitor.requests == "HEAD": r = self.session.head( task.get_complete_target(), headers=Visitor.headers, proxies=Visitor.proxy, verify=False, timeout=timeout, auth=Visitor.auth, cookies=Visitor.cookies, allow_redirects=Visitor.is_allow_redirects) else: if Visitor.requests == "GET": r = self.session.get( task.get_complete_target(), headers=Visitor.headers, verify=False, timeout=timeout, auth=Visitor.auth, cookies=Visitor.cookies, allow_redirects=Visitor.is_allow_redirects) elif Visitor.requests == "HEAD": r = self.session.head( task.get_complete_target(), headers=Visitor.headers, verify=False, timeout=timeout, auth=Visitor.auth, cookies=Visitor.cookies, allow_redirects=Visitor.is_allow_redirects) after = time.time() delta = (after - now) * 1000 tmp_content = r.content task.response_size = len(tmp_content) task.response_time = delta task.set_response_code(r.status_code) self.__time.append(delta) # If discriminator is found we mark it 404 if sys.version_info[0] >= 3: tmp_content = tmp_content.decode('Latin-1') if Visitor.discriminator and Visitor.discriminator in tmp_content: task.ignorable = True if Visitor.banned_md5 and hashlib.md5( "".join(tmp_content)).hexdigest() == self.banned_md5: task.ignorable = True # Check if page size is not what we need if task.response_size in Visitor.size_discriminator: task.ignorable = True # Look for interesting content if task.content and (task.content in tmp_content): task.content_has_detected(True) # Look for a redirection if Visitor.is_allow_redirects: if len(r.history) > 0 and not _dumb_redirect( r.history[-1].url): task.response_code = str(r.history[0].status_code) task.location = r.history[-1].url else: if str(r.status_code).startswith('3'): task.set_response_code('404') task.ignorable = True if 'content-type' in [h.lower() for h in r.headers.keys()]: try: task.response_type = r.headers['Content-Type'].split( ';')[0] except: pass self.lock.acquire() Console.body(task) self.results.get_results_queue().put(task) self.results.get_a_task() if Visitor.delay: time.sleep(Visitor.delay) except (requests.ConnectionError, requests.Timeout) as e: # TODO log to a file instead of screen print("[!] Timeout/Connection error") print(e) except Exception as e: print("[!] General exception while visiting") print(e) finally: self.lock.release()