class Run(object):
def __init__(self, loop):
self.proxies = ProxyDB()
if proxy_source:
asyncio.ensure_future(self.get_proxies(), loop=loop)
async def get_proxies(self):
while True:
print("Proxies loading...")
protos = ["http://", "https://"]
if any(p in proxy_source for p in protos):
f = util.get_page
else:
f = util.load_file
result = await f(proxy_source)
self.proxies.add(result.split('\n'))
print("Proxies loaded.")
await asyncio.sleep(10 * 60)
async def work(self):
args = ["--timeout 5"]
if sort_position:
this_position = next(
x for x in positions if x not in used_positions
)
used_positions.append(this_position)
args.extend(
[
"--window-position=%s,%s" % this_position,
"--window-size=400,400",
]
)
options = {"ignoreHTTPSErrors": True, "args": args}
proxy = await self.proxies.get() if proxy_source else None
client = Solver(
pageurl,
sitekey,
options=options,
proxy=proxy
)
try:
async with timeout(180):
result = await client.start()
except CancelledError:
if client.launcher:
if not client.launcher.chromeClosed:
await client.launcher.waitForChromeToClose()
finally:
if sort_position:
used_positions.remove(this_position)
if result:
print(result)
self.proxies.set_active(proxy, False)
if result['status'] == "detected":
self.proxies.set_banned(proxy)
else:
self.proxies.set_used(proxy)
if result['status'] == "success":
return result['code']
async def main(self):
if proxy_source:
while self.proxies is None:
await asyncio.sleep(1)
tasks = [asyncio.ensure_future(self.work()) for i in range(threads)]
completed, pending = await asyncio.wait(
tasks, return_when=asyncio.FIRST_COMPLETED
)
count = 0
while True:
for task in completed:
result = task.result()
if result:
count += 1
print(f"{count}: {result}")
pending.add(asyncio.ensure_future(self.work()))
completed, pending = await asyncio.wait(
pending, return_when=asyncio.FIRST_COMPLETED
)
class Run(object):
proxies_loading = True
def __init__(self, loop):
self.proxies = ProxyDB(last_banned_timeout=45 * 60)
self.loop = loop
if proxy_source:
asyncio.ensure_future(self.get_proxies(), loop=loop)
async def get_proxies(self):
while True:
self.proxies_loading = True
print("Proxies loading...")
protos = ["http://", "https://"]
if any(p in proxy_source for p in protos):
f = util.get_page
else:
f = util.load_file
result = await f(proxy_source)
self.proxies.add(result.split('\n'))
self.proxies_loading = False
print("Proxies loaded.")
await asyncio.sleep(10 * 60)
async def work(self):
args = ["--timeout 5"]
if sort_position:
this_position = next(x for x in positions
if x not in used_positions)
used_positions.append(this_position)
args.extend([
"--window-position=%s,%s" % this_position,
"--window-size=400,400",
])
options = {"ignoreHTTPSErrors": True, "method": 'images', "args": args}
proxy = self.proxies.get() if proxy_source else None
proxy_auth = None
if proxy_username and proxy_password:
proxy_auth = {
"username": proxy_username,
"password": proxy_password
}
client = Solver(pageurl,
sitekey,
options=options,
loop=self.loop,
proxy=proxy,
proxy_auth=proxy_auth)
result = None
try:
async with timeout(180):
result = await client.start()
finally:
if sort_position:
used_positions.remove(this_position)
if result:
self.proxies.set_active(proxy, False)
if result['status'] == "detected":
self.proxies.set_banned(proxy)
else:
if result['status'] == "success":
return result['code']
async def main(self):
if proxy_source:
while not self.proxies_loading:
await asyncio.sleep(1)
tasks = [asyncio.ensure_future(self.work()) for i in range(threads)]
completed, pending = await asyncio.wait(
tasks, return_when=asyncio.FIRST_COMPLETED)
count = 0
while True:
for task in completed:
result = task.result()
if result:
count += 1
print(f"{count}: {result}")
pending.add(asyncio.ensure_future(self.work()))
completed, pending = await asyncio.wait(
pending, return_when=asyncio.FIRST_COMPLETED)