Ejemplo n.º 1
0
def update_proxy(start_date, end_date):
    """获取最新的代理,并且验证代理
    """
    # itmop
    itmop_proxys = itmop.get_proxys(start_date, end_date)
    print "itmop_proxys update:", len(itmop_proxys)

    # youdaili
    youdaili_proxys = youdaili.get_proxys(start_date, end_date)
    print "youdaili_proxys update:", len(youdaili_proxys)

    # 历史
    proxy_dat_file_path = main_config.PROXY_DAT_FILE_PATH
    history_proxys = util.read_proxys(proxy_dat_file_path) \
        if os.path.exists(proxy_dat_file_path) else set()
    print "read history proxy:", len(history_proxys)

    # 合并
    proxy_list = list(youdaili_proxys.union(history_proxys).union(itmop_proxys))
    print "union history, itmop, youdaili proxys:", len(proxy_list)

    # 验证
    print "start to validate proxys"
    success_pers = validate.validate_proxys(
        proxy_list, max_clients=main_config.MAX_CLIENTS,
        interval=main_config.INTERVAL)
    print "end to validate proxys"

    # 持久化代理
    util.write_proxys([proxy_list[i] for i, success_per in enumerate(success_pers)
                       if success_per >= main_config.PROXY_THRESHOLD],
                      proxy_dat_file_path)
    print "write proxy into file end"
Ejemplo n.º 2
0
def validate_proxy():
    """验证代理
    """
    # 获取本地代理
    proxy_dat_file_path = main_config.PROXY_DAT_FILE_PATH
    history_proxys = util.read_proxys(proxy_dat_file_path) \
        if os.path.exists(proxy_dat_file_path) else set()
    print "read history proxy:", len(history_proxys)

    # 验证代理
    proxy_list = list(history_proxys)
    print "start to validate proxys"
    success_pers = validate.validate_proxys(
        proxy_list, max_clients=main_config.MAX_CLIENTS,
        interval=main_config.INTERVAL)
    print "end to validate proxys:", len(proxy_list)

    # 持久化代理
    util.write_proxys([proxy_list[i] for i, success_per in enumerate(success_pers)
                       if success_per >= main_config.PROXY_THRESHOLD],
                      proxy_dat_file_path)
    print "write proxy into file end"
Ejemplo n.º 3
0
def update_proxy(start_date, end_date):
    """获取最新的代理,并且验证代理
    """
    # itmop
    itmop_proxys = itmop.get_proxys(start_date, end_date)
    print "itmop_proxys update:", len(itmop_proxys)

    # youdaili
    youdaili_proxys = youdaili.get_proxys(start_date, end_date)
    print "youdaili_proxys update:", len(youdaili_proxys)

    # 历史
    proxy_dat_file_path = main_config.PROXY_DAT_FILE_PATH
    history_proxys = util.read_proxys(proxy_dat_file_path) \
        if os.path.exists(proxy_dat_file_path) else set()
    print "read history proxy:", len(history_proxys)

    # 合并
    proxy_list = list(
        youdaili_proxys.union(history_proxys).union(itmop_proxys))
    print "union history, itmop, youdaili proxys:", len(proxy_list)

    # 验证
    print "start to validate proxys"
    success_pers = validate.validate_proxys(
        proxy_list,
        max_clients=main_config.MAX_CLIENTS,
        interval=main_config.INTERVAL)
    print "end to validate proxys"

    # 持久化代理
    util.write_proxys([
        proxy_list[i] for i, success_per in enumerate(success_pers)
        if success_per >= main_config.PROXY_THRESHOLD
    ], proxy_dat_file_path)
    print "write proxy into file end"
Ejemplo n.º 4
0
def validate_proxy():
    """验证代理
    """
    # 获取本地代理
    proxy_dat_file_path = main_config.PROXY_DAT_FILE_PATH
    history_proxys = util.read_proxys(proxy_dat_file_path) \
        if os.path.exists(proxy_dat_file_path) else set()
    print "read history proxy:", len(history_proxys)

    # 验证代理
    proxy_list = list(history_proxys)
    print "start to validate proxys"
    success_pers = validate.validate_proxys(
        proxy_list,
        max_clients=main_config.MAX_CLIENTS,
        interval=main_config.INTERVAL)
    print "end to validate proxys:", len(proxy_list)

    # 持久化代理
    util.write_proxys([
        proxy_list[i] for i, success_per in enumerate(success_pers)
        if success_per >= main_config.PROXY_THRESHOLD
    ], proxy_dat_file_path)
    print "write proxy into file end"