async def process_raw_proxy(self, proxy, collector_id): self.logger.debug("processing raw proxy \"{}\"".format(proxy)) try: _, auth_data, domain, port = proxy_validator.retrieve(proxy) except proxy_validator.ValidationError as ex: self.collectors_logger.error( "Collector with id \"{}\" returned bad raw proxy \"{}\". " "Message: {}".format(collector_id, proxy, ex) ) return # don't care about protocol try: proxy = await db.get( Proxy.select().where( Proxy.auth_data == auth_data, Proxy.domain == domain, Proxy.port == port, ) ) if proxy.last_check_time + settings.PROXY_NOT_CHECKING_PERIOD >= time.time(): proxy_short_address = "" if auth_data: proxy_short_address += auth_data + "@" proxy_short_address += "{}:{}".format(domain, port) self.logger.debug( "skipping proxy \"{}\" from collector \"{}\"".format( proxy_short_address, collector_id) ) return except Proxy.DoesNotExist: pass for raw_protocol in range(len(Proxy.PROTOCOLS)): while not self.good_proxies_are_processed: # TODO: find a better way await asyncio.sleep(0.1) new_proxy = Proxy() new_proxy.raw_protocol = raw_protocol new_proxy.auth_data = auth_data new_proxy.domain = domain new_proxy.port = port await self.add_proxy_to_queue(new_proxy, collector_id)
if len(sys.argv) < 2: print("Usage: python3 {} RESULT_FILE".format( sys.argv[0] if len(sys.argv) == 1 else "program")) exit(1) last_check_time = int(time.time()) i = 0 with open(sys.argv[1], 'r') as file: for line in file: try: print("line {}".format(i)) json_proxy = json.loads( base64.b64decode(line.encode()).decode()) # print(json_proxy) proxy = Proxy() proxy.raw_protocol = Proxy.PROTOCOLS.index( json_proxy['protocol']) proxy.auth_data = "" proxy.domain = json_proxy['domain'] proxy.port = json_proxy['port'] proxy.last_check_time = last_check_time last_check_time += 1 proxy.number_of_bad_checks = settings.REMOVE_ON_N_BAD_CHECKS - 5 session.add(proxy) session.commit() except sqlalchemy.exc.IntegrityError: session.rollback() print("proxy {} exists".format(proxy)) i += 1