async def start_check(self, loop, delete_invalid_proxies=True):
tasks = []
for i in self.sqlConn.fetch_one_row():
if i is not None:
tasks.append(self.__check(i[1], delete_invalid_proxies))
if len(tasks) <= 0:
CliMessage.print_with_status('No tasks to check!',
Color.Yellow,
status='warning')
return
await asyncio.gather(*tasks)
def connect_db(self):
if self.is_open:
return False
if self.overwrite:
if os.path.exists(self.database_path):
os.unlink(self.database_path)
try:
self.conn = sqlite3.connect(self.database_path)
except Exception as e:
CliMessage.print_with_status(e.args, Color.Red, Color.Bold, None,
'failed')
CliMessage.print_with_status(
f"Failed to connect SQLite database...", Color.Red, Color.Bold,
None, 'failed')
exit(-1)
self.is_open = True
self.cursor = self.conn.cursor()
def signal_callback(self,*args):
"""
reveice SIGINT signal and exit
"""
self.q_ips.task_done()
self.q_ips.queue.clear()
if hasattr(self,'sqlConn'):
if self.sqlConn:
del self.sqlConn
if self._async:
try:
self.loop.stop()
self.loop.close()
except:
pass
CliMessage.print_with_status('Exit...',Color.Red,Color.Bold,None,'failed')
exit(0)
def connect_db(self):
if self.is_open:
return False
try:
self.conn = pymysql.Connection(host=self.host,
user=self.user,
password=self.pwd,
port=self.port,
charset='utf8')
except pymysql.MySQLError as e:
CliMessage.print_with_status(e.args, Color.Red, Color.Bold, None,
'failed')
CliMessage.print_with_status(f"Failed to connect MySQL Server...",
Color.Red, Color.Bold, None, 'failed')
exit(-1)
self.is_open = True
self.cursor = self.conn.cursor()
return True
def main(help_panel):
# examples:
# proxyGet.py --help
# proxyGet.py -0 -5
# proxyGet.py -3 -7 --async
# proxyGet.py -0 -7 -m 10 -t 5 --mysql --host=127.0.0.1 --port=3306 -u root -p xxx --database=db_proxy --table=tb_proxy
# proxyGet.py -1 -5 --async --max=2 --timeout=3 --retry=1 --file=db/test.db --table=tb_proxy --from=2 --to=4
from_page_index = help_panel['from_page_index']
to_page_index = help_panel['to_page_index']
if from_page_index <= 0 or to_page_index <= 0 or from_page_index > to_page_index:
CliMessage.print_with_status('Page index is invalid', Color.Red,
Color.Bold, None, 'failed')
exit(0)
col = help_panel.collect(help_panel.url)
if help_panel['_async']:
CliMessage.print_with_status(
'Currently you have selected an asynchronous operation to obtain the proxy',
Color.Green)
testpool = asyncTestProxyPool(col.table_field(), help_panel.optionBox)
else:
testpool = TestProxyPool(col.table_field(), help_panel.optionBox)
for page_index in range(from_page_index, to_page_index + 1):
# set new page index
col.set_page_index(page_index)
for per_proxy in col.run():
if not testpool.add_ip(per_proxy):
break
try:
testpool.start_test()
except:
break
if page_index != to_page_index:
CliMessage.print_with_status('Next Page...')
async def _async_start_test(self):
while not self.q_ips.empty():
proxyIPinfo=self.q_ips.get()
per_proxies = proxyIPinfo.proxies
if self.sqlConn.find_exist(per_proxies):
continue
proxies = 'http://'+per_proxies
CliMessage.print_with_status(f'Testing {per_proxies}', Color.Green)
_retry=self.retry
while _retry > 0:
# deal with next IP
ls_ip_info=await self.test_ip(proxies)
if ls_ip_info==200:
CliMessage.print_with_status(f'OK! {per_proxies}', Color.Green,Color.Bold)
ls=proxyIPinfo.values()
real_ip_info=self.get_current_proxyip_info(per_proxies)
if real_ip_info and len(real_ip_info)>=3 and ls and len(ls)>=2:
ls[1]=real_ip_info[1]+real_ip_info[2]
self.sqlConn.insert(ls)
else:
self.sqlConn.insert(ls)
break
else:
CliMessage.print_with_status(f'Retrying {per_proxies}', Color.Yellow, None,None,'warning')
_retry-=1
def start_test(self):
while not self.q_ips.empty():
proxyIPinfo=self.q_ips.get()
per_proxies = proxyIPinfo.proxies
if self.sqlConn.find_exist(per_proxies):
continue
proxies = {
'http': 'http://'+per_proxies,
'https': 'http://' + per_proxies
}
CliMessage.print_with_status(f'Testing {per_proxies}', Color.Green)
_retry=self.retry
while _retry > 0:
# Test IP
ls_ip_info=self.test_ip(proxies)
if ls_ip_info==200:
CliMessage.print_with_status(f'OK!', Color.Green,Color.Bold)
ls=proxyIPinfo.values()
real_ip_info=self.get_current_proxyip_info(per_proxies)
if not self.sqlConn.find_exist(per_proxies):
if real_ip_info and isinstance(real_ip_info,list):
ls[1]=real_ip_info[1]+real_ip_info[2]
self.sqlConn.insert(ls)
else:
self.sqlConn.insert(ls)
break
else:
CliMessage.print_with_status(f'Retrying...', Color.Yellow, None,None,'warning')
_retry-=1
self.cur_size=0
async def __check(self, proxy, delete_invalid_proxies):
for p in range(2):
try:
async with aiohttp.ClientSession(
headers=self.headers) as session:
async with session.get(self.url,
timeout=self.timeout,
proxy='http://' + proxy,
verify_ssl=self.verify) as resp:
self.status_code = resp.status
break
except:
self.status_code = 0
if self.status_code == 200:
self.index += 1
self.successed_count += 1
CliMessage.print_with_status(f'Good.{self.index} http://{proxy}',
Color.Green)
else:
self.failed_count += 1
if delete_invalid_proxies:
if not self.sqlConn.is_closed():
self.sqlConn.remove(value=proxy)
def done(self, task):
CliMessage.print_with_status('Done', Color.Green, Color.Bold)
CliMessage.print_with_status(
f'Successful prixies: {self.successed_count}', Color.Green,
Color.Bold)
CliMessage.print_with_status(f'Failed prixies: {self.failed_count}',
Color.Red,
Color.Bold,
status='failed')
def main():
parser = OptionParser('proxyChecker -h/-d/-r')
parser.set_description(
'proxyChecker check the proxies and remove invalid proxies from database.'
)
parser.add_option('-t',
action='store',
metavar='SEC',
default=6,
dest='timeout',
help='connection timeout [:6]',
type='int')
parser.add_option('-d',
action='store_true',
dest='delete',
help='delete invalid proxies')
parser.add_option('-r',
action='store_true',
dest='run_forever',
help='run forever but wait for 5 seconds')
(opt, args) = parser.parse_args()
if proxy_config.SQL_TYPE == 'sqlite':
sqlConn = proxySQLite(proxy_config.DATABASE_PATH, proxy_config.TABLE)
else:
sqlConn = proxyMySQL(proxy_config.MYSQL_USER,
proxy_config.MYSQL_PWD,
host=proxy_config.MYSQL_HOST,
port=proxy_config.MYSQL_PORT,
database_name=proxy_config.DATABASE,
table_name=proxy_config.TABLE)
delete_invalid_proxies = False
if opt.delete:
CliMessage.print_with_status(
"proxyChecker will delete invalid proxies!",
Color.Yellow,
status='warning')
delete_invalid_proxies = True
loop = asyncio.get_event_loop()
# run forever
while True:
checker = ProxyChecker(sqlConn, timeout=opt.timeout)
try:
gather = asyncio.gather(
checker.start_check(loop, delete_invalid_proxies))
gather.add_done_callback(checker.done)
loop.run_until_complete(gather)
except KeyboardInterrupt:
CliMessage.print_with_status('Exit......',
Color.Red,
Color.Bold,
status='failed')
if sqlConn.is_closed():
loop.close()
exit(0)
finally:
# database was closed
if sqlConn.is_closed():
CliMessage.print_with_status('Database was closed!',
Color.Red,
Color.Bold,
status='failed')
loop.close()
exit(0)
if not opt.run_forever:
break
time.sleep(5)
loop.close()
sqlConn.close()
def parser_options(self):
self.__optBox.add_key_value('proxy_type_xila',self.__opt.proxy_type_xila)
self.__optBox.add_key_value('proxy_type_nima',self.__opt.proxy_type_nima)
self.__optBox.add_key_value('proxy_type_xici',self.__opt.proxy_type_xici)
self.__optBox.add_key_value('proxy_type_kuai',self.__opt.proxy_type_kuai)
self.__optBox.add_key_value('proxy_type_yip7',self.__opt.proxy_type_yip7)
self.__optBox.add_key_value('proxy_common',self.__opt.proxy_common)
self.__optBox.add_key_value('proxy_high_anonymous',self.__opt.proxy_high_anonymous)
self.__optBox.add_key_value('proxy_http',self.__opt.proxy_http)
self.__optBox.add_key_value('proxy_https',self.__opt.proxy_https)
self.__optBox.add_key_value('max_proxies',self.__opt.max_proxies)
self.__optBox.add_key_value('retry',self.__opt.retry)
self.__optBox.add_key_value('timeout',self.__opt.timeout)
self.__optBox.add_key_value('author',self.__opt.author)
self.__optBox.add_key_value('version',self.__opt.version)
self.__optBox.add_key_value('overwrite',self.__opt.overwrite)
self.__optBox.add_key_value('from_page_index',self.__opt.from_page_index)
self.__optBox.add_key_value('to_page_index',self.__opt.to_page_index)
self.__optBox.add_key_value('_async',self.__opt._async)
self.__optBox.add_key_value('sqlite3',self.__opt.sqlite3)
self.__optBox.add_key_value('mysql',self.__opt.mysql)
self.__optBox.add_key_value('mysql_host',self.__opt.mysql_host)
self.__optBox.add_key_value('mysql_port',self.__opt.mysql_port)
self.__optBox.add_key_value('mysql_user',self.__opt.mysql_user)
self.__optBox.add_key_value('mysql_pwd',self.__opt.mysql_pwd)
self.__optBox.add_key_value('mysql_db',self.__opt.mysql_db)
self.__optBox.add_key_value('mysql_sqlite_tb',self.__opt.mysql_sqlite_tb)
self.__optBox.add_key_value('sqlite_file_path',self.__opt.sqlite_file_path)
if self.__optBox['author']:
self.display_author_info()
exit(0)
if self.__optBox['version']:
CliMessage.print_with_status(f"proxyGet version: v{self.version}")
exit(0)
if self.__optBox['proxy_type_xila']:
self.interface = proxyServer.XilaProxy
self.collect = collectProxyList.Xila
elif self.__optBox['proxy_type_nima']:
self.interface = proxyServer.NimaProxy
self.collect = collectProxyList.Nima
elif self.__optBox['proxy_type_xici']:
self.interface = proxyServer.XiciProxy
self.collect = collectProxyList.Xici
elif self.__optBox['proxy_type_kuai']:
self.interface = proxyServer.KuaiProxy
self.collect = collectProxyList.Kuai
elif self.__optBox['proxy_type_yip7']:
self.interface = proxyServer.Yip7Proxy
self.collect = collectProxyList.Yip7
else:
CliMessage.print_with_status('please choose a website type!', Color.Red, None, None, 'failed')
exit(-1)
if self.interface != proxyServer.Yip7Proxy and self.interface != proxyServer.KuaiProxy:
if self.__optBox['proxy_common']:
self.url = self.interface().commmon_proxy()
elif self.__optBox['proxy_high_anonymous']:
self.url = self.interface().high_anonymous_proxy()
elif self.__optBox['proxy_http']:
self.url = self.interface().http_proxy()
elif self.__optBox['proxy_https']:
self.url = self.interface().https_proxy()
else:
CliMessage.print_with_status('please choose a proxy type!',Color.Red,None,None,'failed')
exit(-1)
elif self.interface==proxyServer.KuaiProxy:
if self.__optBox['proxy_common']:
self.url = self.interface().commmon_proxy()
elif self.__optBox['proxy_high_anonymous']:
self.url = self.interface().high_anonymous_proxy()
else:
CliMessage.print_with_status('please choose a proxy type!', Color.Red, None, None, 'failed')
exit(-1)
else:
if self.__optBox['proxy_http']:
self.url = self.interface().http_proxy()
else:
CliMessage.print_with_status('please choose a proxy type!',Color.Red,None,None,'failed')
exit(-1)
def output_logo(self):
CliMessage.print(Logo.text(),Color.random_fgcolor(),Color.Bold)
def display_author_info(self):
CliMessage.print_with_status(f'Author: {self.author}',Color.Cyan)
CliMessage.print_with_status(f'Blog: {self.blog}',Color.Green)