# @datetime: 7/19 0019 # @author: [email protected] # @description: """ 爬虫的相关工具,包括html下载器 """ import json import time import requests from util import logger, commonUtil Logger = logger.ConsoleLogger(commonUtil.get_file_name(__file__)).getLogger() class HtmlDownloader(object): __type_get = "get" __type_post = "post" __default_delay = 5 def __init__(self, request_header=None, request_cookies=None, proxies=None, delay=False,turn_to_json = False): """html页面下载器,通过requests模块的request方法进行页面下载,返回response.text :param request_header: :param request_cookies: :param proxies: :param delay: 是否设置延迟,可传入boolean或者int类型,为True时默认值为 :param turn_to_json: 结果是否转换为json """
def test_get_file_name(self): filepath = "W:/code/git/myProject/pythonStudyRoad/study/dasktop_ui/wx_study.py" self.assertEqual("wx_study", commonUtil.get_file_name(filepath, separate="/"))