def __init__(self):
self.proxies = ProxyHandler()
self.tasker = Tasker()
self.saver = Saver()
self.log_handler = LogHandler()
self.tieba_task = tieba_task
self.loop_num = LOOP_NUM
class Scheduler(object):
def __init__(self):
self.proxies = ProxyHandler()
self.tasker = Tasker()
self.saver = Saver()
self.log_handler = LogHandler()
self.tieba_task = tieba_task
self.loop_num = LOOP_NUM
def end_task(self, loop):
# 保存数据到mongodb
self.saver.run()
if self.tasker.check_retries():
self.retry_task(loop)
else:
loop.close()
self.log_handler.logger().info('完成循环')
print('Finish!!')
def retry_task(self, loop):
#已经分片好的每次重试的url列表
per_step_urls_list = self.tasker.get_perloop_retry()
# 获取代理池中的代理
proxy_list = self.proxies.get_proxies()
for per_step_urls in per_step_urls_list:
# 待处理
tasks = tieba_task('retry', per_step_urls, proxy_list)
# 启动
loop.run_until_complete(asyncio.gather(*tasks))
self.end_task(loop)
# @run_time
def start_task(self):
loop = asyncio.get_event_loop()
# 获取代理池中的代理
proxy_list = self.proxies.get_proxies()
# 每次所有任务
one_task = self.tasker.get_task()
# self.loop_num: 每个任务循环几次, 避免Semaphore量太大
for index in range(self.loop_num):
# 每次循环的urls
per_step_urls = self.tasker.get_perloop(one_task)
# 待处理
tasks = tieba_task(index, per_step_urls, proxy_list)
# 启动
loop.run_until_complete(asyncio.gather(*tasks))
self.end_task(loop)
def main():
parser = argparse.ArgumentParser(
description='Penta is Pentest automation tool')
parser.add_argument("-target",
dest="target",
help="Specify target IP / domain")
parser.add_argument(
"-ports",
dest="ports",
help=
"Specify the target port(s) separated by comma. Default: 21,22,25,80,110,443,8080",
default="21,22,25,80,110,443,8080")
parser.add_argument("-proxy", dest="proxy", help="Proxy[IP:PORT]")
options = parser.parse_args()
checker = Inspect()
nmap_scan = NmapScanner()
dns_scan = DnsScanner()
shodan_search = ShodanSearch()
ftp_access = FtpConnector()
ssh_access = SshConnector()
msf_scan = MetaSploitRPC()
log_handler = LogHandler()
hostname = ""
num_menu = ""
if options.target is None:
while hostname == "":
hostname = input("[*] Specify IP or name domain:")
else:
hostname = options.target
print("[*] Get IP address from host name...")
ip = socket.gethostbyname(hostname)
print('[+] The IP address of {} is {}{}{}\n'.format(
hostname, Colors.GREEN, ip, Colors.END))
while num_menu != 0:
num_menu = menu()
if num_menu == "0":
sys.exit(1)
elif num_menu == "1":
port_list = options.ports.split(',')
for port in port_list:
nmap_scan.nmap_scan(ip, port)
results = nmap_scan.nmap_json_export(ip, options.ports)
log_filename = "scan_{}.json".format(hostname)
log_handler.save_logfile(log_filename, results)
print("[+] {}{}{} was generated".format(Colors.GREEN, log_filename,
Colors.END))
print("\n")
elif num_menu == "2":
nmap_scan.nmap_menu(ip)
print("\n")
elif num_menu == "3":
print("\n")
checker.check_option_methods(hostname)
print("\n")
elif num_menu == "4":
print("\n")
dns_scan.check_dns_info(ip, hostname)
print("\n")
elif num_menu == "5":
shodan_search.shodan_host_info(ip)
print("\n")
elif num_menu == "6":
ftp_access.ftp_connect_anonymous(ip)
print("\n")
elif num_menu == "7":
ssh_access.ssh_connect(ip)
print("\n")
elif num_menu == "8":
msf_scan.msf_scan(ip)
print("\n")
elif num_menu == "9":
# TODO: hydra brute force login --> smb ssh ftp http
# TODO: malware detect functions e.g avast socks
pass
elif num_menu == "99":
hostname = input("[*] Specify IP or name domain:")
print("[*] Get IP address from host name...")
ip = socket.gethostbyname(hostname)
print('[+] The IP address of {} is {}{}{}\n'.format(
hostname, Colors.GREEN, ip, Colors.END))
else:
print("[-] Incorrect option")
from logging import ERROR, INFO
from itertools import count
from xml.dom.minidom import parse
from urllib2 import urlopen
import scrapy
from crawler.items import PlayerIdItem
from utils import LogHandler
#constants
CURRENT_PLAYERS_SITEMAP = 'http://www.nba.com/current_players.xml'
LOGGER = LogHandler(__name__)
class PlayerIdSpider(scrapy.Spider):
"""
Scrapes current NBA players' ids from their nba.com page.
Attributes:
name (str): Name of spider
allow_domains (List[str]): Domains which spider is allowed to scrape
current_players (List[str]): Nba.com urls of current NBA players
start_urls (List[str]): Urls which spider will scrape
"""
name = 'PlayerIdSpider'
allow_domains = ['http://nba.com/']
current_players = parse(urlopen(CURRENT_PLAYERS_SITEMAP)).getElementsByTagName('loc')
start_urls = [p.childNodes[0].data for p in current_players]
def test_log_handler():
"""
test function LogHandler in Util/LogHandler
:return:
"""
log = LogHandler('test')
log.info('this is a log from test')
log.rename(name='test1')
log.info('this is a log from test1')
log.rename(name='test2')
log.info('this is a log from test2')
class BaseFetcher(ABC):
name = None
enabled = True
urls = []
logger = LogHandler('Provider')
use_proxy = False
@staticmethod
def sleep(sec):
def decorator(func): # 装饰器核心，以被装饰的函数对象为参数，返回装饰后的函数对象
if sec:
def wrapper(self, *args, **kwargs): # 装饰的过程，参数列表适应不同参数的函数
self.lock.acquire()
try:
func(self, *args, **kwargs) # 调用函数
gevent.sleep(sec)
except Exception as e:
self.logger.debug(e)
self.lock.release()
return wrapper
else:
return func
return decorator
def __init__(self, tasks, result, pool=None):
self._tasks = tasks
self._result = result
self.timeout = 5
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.321.132 Safari/537.36',
'Accept': '*/*',
'Pragma': 'no-cache',
'Cache-control': 'no-cache',
'Referer': 'https://www.google.com/'
}
self.pool = pool
self.lock = BoundedSemaphore()
self.prepare() # must run after init
def request(self, url):
resp = self._request(url)
proxies = self.handle(resp)
self.add_result(proxies)
def _request(self, url):
if not Config.proxy:
return Response()
if self.use_proxy:
return requests.get(url,
headers=self.headers,
proxies={
'http': 'http://' + Config.proxy,
'https': 'https://' + Config.proxy
},
timeout=self.timeout)
else:
return requests.get(url,
headers=self.headers,
timeout=self.timeout)
def prepare(self):
pass
def add_task(self, func):
self._tasks.append(func)
def add_result(self, result):
if isinstance(result, (list, set)):
for i in result:
if isinstance(i, tuple):
self._result.add(':'.join(i))
else:
self._result.add(i)
elif isinstance(result, str):
self._result.add(result)
elif isinstance(result, tuple):
self._result.add(':'.join(result))
else:
raise TypeError
@abstractmethod
def handle(self, resp: Response):
pass
def process_urls(self):
return self.urls
def fill_task(self):
urls = self.process_urls()
for url in urls:
task = gevent.spawn(self.request, url)
self.add_task(task)
if self.pool is not None:
self.pool.add(task)
def __str__(self):
return '<Provider name=%s, enabled=%s>' % (self.name, self.enabled)
@classmethod
def test(cls):
tasks = []
result = set()
cls(tasks, result).fill_task()
gevent.joinall(tasks)
print(result)
# coding:utf-8
import time
import gevent
from manager import ProxyManager
from utils import LogHandler
from verify.proxy_verifier import ProxyGeventVerifier
logger = LogHandler('ProxyVerifyGeventScheduler')
class ProxyVerifyGeventScheduler:
def run(self):
try:
proxy_manager = ProxyManager()
logger.info("Start proxy verify")
while True:
proxies = proxy_manager.proxy_verified_before(minutes=30,
limit=1000)
if not len(proxies):
logger.info(
'Not proxy need to be verified! Sleep [ 5 ] minutes.')
proxy_manager.close()
break
verifier = ProxyGeventVerifier()
start = time.time()
tasks = verifier.generate_tasks(proxies)
logger.info('Created %d verify tasks' % len(proxies))
gevent.joinall(tasks)
logger.info('Proxy Verify Using %d sec.' %
import time
from _ssl import SSLError
from json import JSONDecodeError
import gevent
from gevent import socket
from utils._parser import HTTPParseError
from verify.connector import GeventConnector
from model import ProxyModel
from utils import LogHandler
from utils.errors import BadStatusError, RecvTimeout, ConnectTimeout, BadResponseError, ProxySendError
from utils.functions import get_self_ip
from verify.detector import Detector
logger = LogHandler('ProxyGeventVerifier')
class ProxyGeventVerifier:
def __init__(self, post=False, timeout=4):
ext_ip = get_self_ip()
self._method = 'POST' if post else 'GET'
self.http_judge = Detector('http://httpbin.skactor.tk:8080/anything', ext_ip)
self.https_judge = Detector('https://httpbin.skactor.tk/anything', ext_ip)
self._timeout = timeout
def generate_tasks(self, proxies):
tasks = []
for proxy in proxies:
proxy.init(GeventConnector)
task = gevent.spawn(self.check, proxy)
def ip_menu(options):
hostname = ""
num_menu = ""
checker = Inspect()
nmap_scan = NmapScanner()
dns_scan = DnsScanner()
shodan_search = ShodanSearch()
ftp_access = FtpConnector()
ssh_access = SshConnector()
msf_rpc_scan = MetaSploitRPC()
log_handler = LogHandler()
if options.target is None:
while hostname == "":
hostname = input("[>] Specify IP or name domain: ")
else:
hostname = options.target
print("[*] Get IP address from host name...")
ip = socket.gethostbyname(hostname)
print('[+] The IP address of {} is {}{}{}\n'.format(hostname, Colors.LIGHTGREEN, ip, Colors.END))
ip_menu_list()
while num_menu != 0:
num_menu = choice_num()
if num_menu == 0:
main_menu(options)
elif num_menu == 1:
port_list = options.ports.split(',')
for port in port_list:
nmap_scan.nmap_scan(ip, port)
results = nmap_scan.nmap_json_export(ip, options.ports)
log_filename = "scan_{}.json".format(hostname)
log_handler.save_logfile(log_filename, results)
print("[+] {}{}{} was generated".format(Colors.LIGHTGREEN, log_filename, Colors.END))
elif num_menu == 2:
nmap_scan.nmap_menu(ip)
elif num_menu == 3:
checker.check_option_methods(hostname)
elif num_menu == 4:
dns_scan.check_dns_info(ip, hostname)
elif num_menu == 5:
shodan_search.shodan_host_info(ip)
elif num_menu == 6:
ftp_access.ftp_connect_anonymous(ip)
elif num_menu == 7:
ssh_access.ssh_connect(ip)
elif num_menu == 8:
msf_rpc_scan.scan(ip)
elif num_menu == 9:
# TODO: hydra brute force login --> smb ssh ftp http
# TODO: malware detect functions e.g avast socks
pass
elif num_menu == 99:
hostname = input("[*] Specify IP or name domain: ")
print("[*] Get IP address from host name...")
ip = socket.gethostbyname(hostname)
print('[+] The IP address of {} is {}{}{}\n'.format(hostname, Colors.LIGHTGREEN, ip, Colors.END))
else:
logging.error("Incorrect option")
ip_menu_list()
# coding:utf-8
import datetime
from sqlalchemy import asc, desc, update
from model import IPLocation
from model import ProxyModel
from utils import Config, LogHandler
logger = LogHandler('ProxyManager')
class ProxyManager:
def __init__(self):
self.session = Config.Session()
def get_proxy(self, proxy: ProxyModel) -> ProxyModel:
return self.session.query(ProxyModel).get(proxy.unique_id)
def get_iploc(self, ip) -> IPLocation:
return self.session.query(IPLocation).get(ip)
def all_proxy(self):
return self.session.query(ProxyModel).all()
def all_iploc(self):
return self.session.query(IPLocation).all()
def all_usable_proxy(self):
return self.session.query(ProxyModel).filter(
ProxyModel.usable == 1).all()
import struct
from abc import ABC, abstractmethod
from socket import inet_aton
from utils import LogHandler
from utils.errors import BadStatusError, BadResponseError
__all__ = ['Socks5Ngtr', 'Socks4Ngtr', 'Connect80Ngtr', 'Connect25Ngtr',
'HttpsNgtr', 'HttpNgtr', 'NGTRS']
SMTP_READY = 2201
logger = LogHandler('Negotiator')
def _CONNECT_request(host, port, **kwargs):
kwargs.setdefault('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.321.132 Safari/537.36')
kw = {
'host': host,
'port': port,
'headers': '\r\n'.join(('%s: %s' % (k, v) for k, v in kwargs.items()))
}
req = 'CONNECT {host}:{port} HTTP/1.1\r\nHost: {host}\r\n{headers}\r\nConnection: keep-alive\r\n\r\n'.format(**kw).encode()
return req
class BaseNegotiator(ABC):
"""Base Negotiator."""
name = None
check_anon_lvl = False
# coding:utf-8
from gevent import ssl, socket
from verify.connector.base_connector import BaseConnector
from utils import LogHandler
from utils.errors import RecvTimeout, ConnectTimeout, ProxySendError
from utils.response import HTTPSocketResponse, HTTPConnectionClosed
from verify.negotiators import NGTRS
logger = LogHandler('GeventConnector')
class GeventConnector(BaseConnector):
def __init__(self, protocol, ip, port, timeout=5):
super(GeventConnector, self).__init__(protocol, ip, port, timeout)
self._socket = {'conn': None, 'ssl': None}
self.negotiator = NGTRS[self.protocol.upper()](self)
self.use_full_path = self.negotiator.use_full_path
self.parser = None
def negotiate(self, judge):
self.negotiator.negotiate(judge)
@property
def socket(self):
return self._socket.get('ssl') or self._socket.get('conn')
@socket.setter
def socket(self, value):
if self._socket.get('conn'):
# coding:utf-8
import gc
import time
import gevent
from manager import ProxyFetcherManager, ProxyManager
from model import ProxyModel
from utils import LogHandler
from verify.proxy_verifier import ProxyGeventVerifier
gevent.hub.Hub.NOT_ERROR = (Exception, )
logger = LogHandler('ProxyFetcherScheduler')
class ProxyFetcherScheduler:
def __gen_fetch_tasks(self, tasks, result):
start = time.time()
for FetcherClass in ProxyFetcherManager.fetchers():
if not FetcherClass.enabled:
continue
fetcher = FetcherClass(tasks, result)
logger.info("Start Fetcher: %s" % fetcher)
fetcher.fill_task()
logger.debug('Using %.2f seconds to start fetchers' %
(time.time() - start))
def __wait_fetch(self, tasks):
start = time.time()
gevent.joinall(tasks)
logger.debug('Using %.2f second to finish fetch processes' %
