def create_or_update_proxy(raw_protocol: Proxy.PROTOCOLS, auth_data, domain, port, start_checking_time, end_checking_time): if raw_protocol is None or domain is None or port is None or auth_data is None or start_checking_time is None\ or end_checking_time is None: raise Exception("Bad arguments") if raw_protocol < 0 or raw_protocol >= len(Proxy.PROTOCOLS): raise Exception("Bad protocol") response_time = int( round((end_checking_time - start_checking_time) * 1000000)) proxy = session.query(Proxy).filter( sqlalchemy.and_(Proxy.raw_protocol == raw_protocol, Proxy.auth_data == auth_data, Proxy.domain == domain, Proxy.port == port)).first() if proxy: # exists, so update pass else: # doesn't exist, so create proxy = Proxy(number_of_bad_checks=0, raw_protocol=raw_protocol, auth_data=auth_data, domain=domain, port=port) session.add(proxy) if proxy.bad_proxy or proxy.uptime is None or proxy.uptime == 0: proxy.uptime = int(time.time()) if proxy.bad_uptime is None or proxy.bad_uptime == 0 or \ proxy.number_of_bad_checks > settings.DEAD_PROXY_THRESHOLD: proxy.bad_uptime = int(time.time()) proxy.response_time = response_time proxy.number_of_bad_checks = 0 proxy.last_check_time = int(time.time()) checking_time = int(end_checking_time - start_checking_time) if checking_time > settings.PROXY_CHECKING_TIMEOUT: checking_time = settings.PROXY_CHECKING_TIMEOUT proxy.checking_period = \ settings.MIN_PROXY_CHECKING_PERIOD \ + (checking_time / settings.PROXY_CHECKING_TIMEOUT) \ * (settings.MAX_PROXY_CHECKING_PERIOD - settings.MIN_PROXY_CHECKING_PERIOD) session.commit()
if len(sys.argv) < 2: print("Usage: python3 {} RESULT_FILE".format( sys.argv[0] if len(sys.argv) == 1 else "program")) exit(1) last_check_time = int(time.time()) i = 0 with open(sys.argv[1], 'r') as file: for line in file: try: print("line {}".format(i)) json_proxy = json.loads( base64.b64decode(line.encode()).decode()) # print(json_proxy) proxy = Proxy() proxy.raw_protocol = Proxy.PROTOCOLS.index( json_proxy['protocol']) proxy.auth_data = "" proxy.domain = json_proxy['domain'] proxy.port = json_proxy['port'] proxy.last_check_time = last_check_time last_check_time += 1 proxy.number_of_bad_checks = settings.REMOVE_ON_N_BAD_CHECKS - 5 session.add(proxy) session.commit() except sqlalchemy.exc.IntegrityError: session.rollback() print("proxy {} exists".format(proxy)) i += 1