Esempio n. 1
0
    def create_or_update_proxy(raw_protocol: Proxy.PROTOCOLS, auth_data,
                               domain, port, start_checking_time,
                               end_checking_time):
        if raw_protocol is None or domain is None or port is None or auth_data is None or start_checking_time is None\
                or end_checking_time is None:
            raise Exception("Bad arguments")

        if raw_protocol < 0 or raw_protocol >= len(Proxy.PROTOCOLS):
            raise Exception("Bad protocol")

        response_time = int(
            round((end_checking_time - start_checking_time) * 1000000))

        proxy = session.query(Proxy).filter(
            sqlalchemy.and_(Proxy.raw_protocol == raw_protocol,
                            Proxy.auth_data == auth_data,
                            Proxy.domain == domain,
                            Proxy.port == port)).first()

        if proxy:
            # exists, so update
            pass
        else:
            # doesn't exist, so create
            proxy = Proxy(number_of_bad_checks=0,
                          raw_protocol=raw_protocol,
                          auth_data=auth_data,
                          domain=domain,
                          port=port)
            session.add(proxy)

        if proxy.bad_proxy or proxy.uptime is None or proxy.uptime == 0:
            proxy.uptime = int(time.time())

        if proxy.bad_uptime is None or proxy.bad_uptime == 0 or \
                proxy.number_of_bad_checks > settings.DEAD_PROXY_THRESHOLD:
            proxy.bad_uptime = int(time.time())

        proxy.response_time = response_time
        proxy.number_of_bad_checks = 0
        proxy.last_check_time = int(time.time())

        checking_time = int(end_checking_time - start_checking_time)
        if checking_time > settings.PROXY_CHECKING_TIMEOUT:
            checking_time = settings.PROXY_CHECKING_TIMEOUT

        proxy.checking_period = \
            settings.MIN_PROXY_CHECKING_PERIOD \
            + (checking_time / settings.PROXY_CHECKING_TIMEOUT) \
            * (settings.MAX_PROXY_CHECKING_PERIOD - settings.MIN_PROXY_CHECKING_PERIOD)

        session.commit()
Esempio n. 2
0
    if len(sys.argv) < 2:
        print("Usage: python3 {} RESULT_FILE".format(
            sys.argv[0] if len(sys.argv) == 1 else "program"))
        exit(1)

    last_check_time = int(time.time())
    i = 0
    with open(sys.argv[1], 'r') as file:
        for line in file:
            try:
                print("line {}".format(i))
                json_proxy = json.loads(
                    base64.b64decode(line.encode()).decode())
                # print(json_proxy)
                proxy = Proxy()
                proxy.raw_protocol = Proxy.PROTOCOLS.index(
                    json_proxy['protocol'])
                proxy.auth_data = ""
                proxy.domain = json_proxy['domain']
                proxy.port = json_proxy['port']
                proxy.last_check_time = last_check_time
                last_check_time += 1
                proxy.number_of_bad_checks = settings.REMOVE_ON_N_BAD_CHECKS - 5
                session.add(proxy)
                session.commit()
            except sqlalchemy.exc.IntegrityError:
                session.rollback()
                print("proxy {} exists".format(proxy))

            i += 1