def get_a_task(self):
try:
task = self.queue.get(block=True, timeout=self.timeout)
self.process(task)
self.queue.task_done()
Console.body(task)
return True
except Queue.Empty:
return False
def get_a_task(self):
try:
task = self.queue.get(block=True, timeout=self.timeout)
self.process(task)
self.queue.task_done()
Console.body(task)
return True
except Queue.Empty:
return False
def get_a_task(self, alived):
try:
task = self.queue.get(False)
if task:
self.process(task)
self.queue.task_done()
Console.body(task)
else:
print("no task")
return True
except Queue.Empty:
return bool(alived)
def get_a_task(self, alived):
try:
task = self.queue.get(False)
if task:
self.process(task)
self.queue.task_done()
Console.body(task)
else:
print "no task"
return True
except Queue.Empty:
if not alived:
return False
else:
return True
#
# Create the thread_pool and start the daemonized threads
#
thread_pool = []
for visitor_id in range(0, threads):
v = Visitor(visitor_id, payload_queue, manager)
thread_pool.append(v)
v.daemon = True
v.start()
#
# Run the main thread until manager exhaust all tasks
#
time_before_running = time.time()
Console.header()
while True:
try:
if not manager.get_a_task() and any([visitor.is_alive() for visitor in thread_pool]):
print("Finishing...")
break
except KeyboardInterrupt:
print (os.linesep + "Waiting for threads to stop...")
Visitor.kill()
break
time_after_running = time.time()
delta = round(timedelta(seconds=(time_after_running - time_before_running)).total_seconds(), 2)
print("Task took %i seconds" % delta)
)
sys.exit()
# Select if full path is prefered
full_path = args.full_path
# Select if user wants content-type print
show_content_type = args.show_content_type
#
# Run the main thread until manager exhaust all tasks
#
time_before_running = time.time()
Console.number_of_requests = payload.get_total_requests()
Console.number_of_threads = threads
Console.start_eta_queue(30)
Console.show_full_path = full_path
Console.show_content_type = show_content_type
Console.header()
Console.set_show_progress(False if args.no_progress else True)
Console.set_show_colors(False if args.no_colors else True)
#
# Create the thread_pool and start the daemonized threads
#
thread_pool = []
for visitor_id in range(threads):
v = Visitor(visitor_id, payload_queue, manager, manager_lock)
thread_pool.append(v)
v.start()
(total_requests, int(total_requests / threads)))
#
# Create the thread_pool and start the daemonized threads
#
thread_pool = []
for visitor_id in range(0, threads):
v = Visitor(visitor_id, payload_queue, manager)
thread_pool.append(v)
v.daemon = True
v.start()
#
# Run the main thread until manager exhaust all tasks
#
time_before_running = time.time()
Console.start_eta_queue(30, total_requests)
Console.header()
try:
while True:
visitors_alive = any([visitor.is_alive() for visitor in thread_pool])
if not manager.get_a_task(visitors_alive):
break
except KeyboardInterrupt:
sys.stdout.write(os.linesep + "Waiting for threads to stop...")
Visitor.kill()
resp = raw_input(os.linesep + "Keep a resume file? [y/N] ")
if resp == 'y':
resumer.set_line(payload_queue.get().get_number())
with open("resume_file_" + time.strftime("%d_%m_%y_%H_%M", time.localtime()), 'w') as f:
pickle.dump(resumer, f)
)
sys.exit()
# Select if full path is prefered
full_path = args.full_path
# Select if user wants content-type print
show_content_type = args.show_content_type
#
# Run the main thread until manager exhaust all tasks
#
time_before_running = time.time()
Console.number_of_requests = payload.get_total_requests()
Console.number_of_threads = threads
Console.start_eta_queue(30)
Console.show_full_path = full_path
Console.show_content_type = show_content_type
Console.header()
#
# Create the thread_pool and start the daemonized threads
#
thread_pool = []
for visitor_id in range(threads):
v = Visitor(visitor_id, payload_queue, manager, manager_lock)
thread_pool.append(v)
v.start()
# Main loop
try:
try:
with open(resume, "rb") as f:
resumer = pickle.load(f)
args = resumer.get_args()
except Exception:
sys.stdout.write("[!] Could not load a correct resume file, sorry.")
sys.exit()
# Target check and preparation
if not args.target:
print("[!] You need to specify a target")
parser.print_help()
sys.exit()
target = prepare_target(args.target)
Console.add_config("{:37} {:>}".format("Target:", target))
recursive = args.recursive
Console.add_config("{:37} {:>}".format("Recursive requests",
"Yes" if recursive else "No"))
# Persistent connections
persist = args.persist
Console.add_config("{:37} {:>}".format("Persistent connections",
"Yes" if persist else "No"))
# Misc options
extension = args.extension.split(",")
threads = int(args.threads)
banned_response_codes = [int(item) for item in args.banned.split(",") if item]
unbanned_response_codes = [
print("[!] Error setting cookies. Review cookie string (key:value,key:value...)")
sys.exit()
# Select if full path is prefered
full_path = args.full_path
# Select if user wants content-type print
show_content_type = args.show_content_type
#
# Run the main thread until manager exhaust all tasks
#
time_before_running = time.time()
Console.number_of_requests = payload.get_total_requests()
Console.number_of_threads = threads
Console.start_eta_queue(30)
Console.show_full_path = full_path
Console.show_content_type = show_content_type
Console.header()
#
# Create the thread_pool and start the daemonized threads
#
thread_pool = []
for visitor_id in range(threads):
v = Visitor(visitor_id, payload_queue, manager, manager_lock)
thread_pool.append(v)
v.start()
# Main loop
try:
#
# Create the thread_pool and start the daemonized threads
#
thread_pool = []
for visitor_id in range(0, threads):
v = Visitor(visitor_id, payload_queue, manager)
thread_pool.append(v)
v.daemon = True
v.start()
#
# Run the main thread until manager exhaust all tasks
#
time_before_running = time.time()
Console.header()
try:
while True:
visitors_alive = any([v.is_alive() for v in thread_pool])
if not manager.get_a_task(visitors_alive):
break
except KeyboardInterrupt:
sys.stdout.write(os.linesep + "Waiting for threads to stop...")
Visitor.kill()
resp = raw_input(os.linesep + "Resume file? (Type 'y' to get) ")
if resp == 'y':
resumer.set_line(payload_queue.get().get_number())
with open(
"resume_file_" +
time.strftime("%d_%m_%y_%H_%M", time.localtime()), 'w') as f:
def visit(self, task):
def _dumb_redirect(url):
origin = "{0}{1}".format(task.target, task.resource)
# Detect redirect to same page but ended with slash
if url == origin:
return True
if url == origin + '/':
return True
# Detect redirect to root
if url == task.target:
return True
return False
try:
if Visitor.user_agent:
Visitor.headers['User-Agent'] = Visitor.user_agent
now = time.time()
timeout = sum(self.__time) / len(self.__time) if self.__time else 10
# Persistent connections
if Visitor.persist:
if not self.session:
self.session = requests.Session()
else:
self.session = requests
r = None
if Visitor.proxy:
if Visitor.requests == "GET":
r = self.session.get(task.get_complete_target(),
headers=Visitor.headers,
proxies=Visitor.proxy,
verify=False,
timeout=timeout,
auth=Visitor.auth,
cookies=Visitor.cookies,
allow_redirects=Visitor.is_allow_redirects)
elif Visitor.requests == "HEAD":
r = self.session.head(task.get_complete_target(),
headers=Visitor.headers,
proxies=Visitor.proxy,
verify=False,
timeout=timeout,
auth=Visitor.auth,
cookies=Visitor.cookies,
allow_redirects=Visitor.is_allow_redirects)
else:
if Visitor.requests == "GET":
r = self.session.get(task.get_complete_target(),
headers=Visitor.headers,
verify=False,
timeout=timeout,
auth=Visitor.auth,
cookies=Visitor.cookies,
allow_redirects=Visitor.is_allow_redirects)
elif Visitor.requests == "HEAD":
r = self.session.head(task.get_complete_target(),
headers=Visitor.headers,
verify=False,
timeout=timeout,
auth=Visitor.auth,
cookies=Visitor.cookies,
allow_redirects=Visitor.is_allow_redirects)
after = time.time()
delta = (after - now) * 1000
tmp_content = r.content
task.response_size = len(tmp_content)
task.response_time = delta
task.set_response_code(r.status_code)
self.__time.append(delta)
# If discriminator is found we mark it 404
if sys.version_info[0] >= 3:
tmp_content = tmp_content.decode('Latin-1')
if Visitor.discriminator and Visitor.discriminator in tmp_content:
task.ignorable = True
if Visitor.banned_md5 and hashlib.md5("".join(tmp_content)).hexdigest() == self.banned_md5:
task.ignorable = True
# Check if page size is not what we need
if task.response_size in Visitor.size_discriminator:
task.ignorable = True
# Look for interesting content
if task.content and (task.content in tmp_content):
task.content_has_detected(True)
# Look for a redirection
if Visitor.is_allow_redirects:
if len(r.history) > 0 and not _dumb_redirect(r.history[-1].url):
task.response_code = str(r.history[0].status_code)
task.location = r.history[-1].url
else:
if str(r.status_code).startswith('3'):
task.set_response_code('404')
task.ignorable = True
if 'content-type' in [h.lower() for h in r.headers.keys()]:
try:
task.response_type = r.headers['Content-Type'].split(';')[0]
except:
pass
self.lock.acquire()
Console.body(task)
self.results.get_results_queue().put(task)
self.results.get_a_task()
if Visitor.delay:
time.sleep(Visitor.delay)
except (requests.ConnectionError, requests.Timeout) as e:
# TODO log to a file instead of screen
print ("[!] Timeout/Connection error")
print (e)
except Exception as e:
print ("[!] General exception while visiting")
print (e)
finally:
self.lock.release()
def put(self, task):
Console.update(task)
self.queue.put(task)
def visit(self, task):
def _dumb_redirect(url):
origin = "{0}{1}".format(task.target, task.resource)
# Detect redirect to same page but ended with slash
if url == origin:
return True
if url == origin + '/':
return True
# Detect redirect to root
if url == task.target:
return True
return False
try:
if Visitor.user_agent:
Visitor.headers['User-Agent'] = Visitor.user_agent
now = time.time()
timeout = sum(self.__time) / len(
self.__time) if self.__time else 10
# Persistent connections
if Visitor.persist:
if not self.session:
self.session = requests.Session()
else:
self.session = requests
r = None
if Visitor.proxy:
if Visitor.requests == "GET":
r = self.session.get(
task.get_complete_target(),
headers=Visitor.headers,
proxies=Visitor.proxy,
verify=False,
timeout=timeout,
auth=Visitor.auth,
cookies=Visitor.cookies,
allow_redirects=Visitor.is_allow_redirects)
elif Visitor.requests == "HEAD":
r = self.session.head(
task.get_complete_target(),
headers=Visitor.headers,
proxies=Visitor.proxy,
verify=False,
timeout=timeout,
auth=Visitor.auth,
cookies=Visitor.cookies,
allow_redirects=Visitor.is_allow_redirects)
else:
if Visitor.requests == "GET":
r = self.session.get(
task.get_complete_target(),
headers=Visitor.headers,
verify=False,
timeout=timeout,
auth=Visitor.auth,
cookies=Visitor.cookies,
allow_redirects=Visitor.is_allow_redirects)
elif Visitor.requests == "HEAD":
r = self.session.head(
task.get_complete_target(),
headers=Visitor.headers,
verify=False,
timeout=timeout,
auth=Visitor.auth,
cookies=Visitor.cookies,
allow_redirects=Visitor.is_allow_redirects)
after = time.time()
delta = (after - now) * 1000
tmp_content = r.content
task.response_size = len(tmp_content)
task.response_time = delta
task.set_response_code(r.status_code)
self.__time.append(delta)
# If discriminator is found we mark it 404
if sys.version_info[0] >= 3:
tmp_content = tmp_content.decode('Latin-1')
if Visitor.discriminator and Visitor.discriminator in tmp_content:
task.ignorable = True
if Visitor.banned_md5 and hashlib.md5(
"".join(tmp_content)).hexdigest() == self.banned_md5:
task.ignorable = True
# Check if page size is not what we need
if task.response_size in Visitor.size_discriminator:
task.ignorable = True
# Look for interesting content
if task.content and (task.content in tmp_content):
task.content_has_detected(True)
# Look for a redirection
if Visitor.is_allow_redirects:
if len(r.history) > 0 and not _dumb_redirect(
r.history[-1].url):
task.response_code = str(r.history[0].status_code)
task.location = r.history[-1].url
else:
if str(r.status_code).startswith('3'):
task.set_response_code('404')
task.ignorable = True
if 'content-type' in [h.lower() for h in r.headers.keys()]:
try:
task.response_type = r.headers['Content-Type'].split(
';')[0]
except:
pass
self.lock.acquire()
Console.body(task)
self.results.get_results_queue().put(task)
self.results.get_a_task()
if Visitor.delay:
time.sleep(Visitor.delay)
except (requests.ConnectionError, requests.Timeout) as e:
# TODO log to a file instead of screen
print("[!] Timeout/Connection error")
print(e)
except Exception as e:
print("[!] General exception while visiting")
print(e)
finally:
self.lock.release()
