async def process_raw_proxy(self, proxy, collector_id):
self.logger.debug("processing raw proxy \"{}\"".format(proxy))
try:
_, auth_data, domain, port = proxy_validator.retrieve(proxy)
except proxy_validator.ValidationError as ex:
self.collectors_logger.error(
"Collector with id \"{}\" returned bad raw proxy \"{}\". "
"Message: {}".format(collector_id, proxy, ex)
)
return
# don't care about protocol
try:
proxy = await db.get(
Proxy.select().where(
Proxy.auth_data == auth_data,
Proxy.domain == domain,
Proxy.port == port,
)
)
if proxy.last_check_time + settings.PROXY_NOT_CHECKING_PERIOD >= time.time():
proxy_short_address = ""
if auth_data:
proxy_short_address += auth_data + "@"
proxy_short_address += "{}:{}".format(domain, port)
self.logger.debug(
"skipping proxy \"{}\" from collector \"{}\"".format(
proxy_short_address, collector_id)
)
return
except Proxy.DoesNotExist:
pass
for raw_protocol in range(len(Proxy.PROTOCOLS)):
while not self.good_proxies_are_processed:
# TODO: find a better way
await asyncio.sleep(0.1)
new_proxy = Proxy()
new_proxy.raw_protocol = raw_protocol
new_proxy.auth_data = auth_data
new_proxy.domain = domain
new_proxy.port = port
await self.add_proxy_to_queue(new_proxy, collector_id)
if len(sys.argv) < 2:
print("Usage: python3 {} RESULT_FILE".format(
sys.argv[0] if len(sys.argv) == 1 else "program"))
exit(1)
last_check_time = int(time.time())
i = 0
with open(sys.argv[1], 'r') as file:
for line in file:
try:
print("line {}".format(i))
json_proxy = json.loads(
base64.b64decode(line.encode()).decode())
# print(json_proxy)
proxy = Proxy()
proxy.raw_protocol = Proxy.PROTOCOLS.index(
json_proxy['protocol'])
proxy.auth_data = ""
proxy.domain = json_proxy['domain']
proxy.port = json_proxy['port']
proxy.last_check_time = last_check_time
last_check_time += 1
proxy.number_of_bad_checks = settings.REMOVE_ON_N_BAD_CHECKS - 5
session.add(proxy)
session.commit()
except sqlalchemy.exc.IntegrityError:
session.rollback()
print("proxy {} exists".format(proxy))
i += 1
