Ejemplo n.º 1
0
# @datetime: 7/19 0019
# @author: [email protected]
# @description:
"""
爬虫的相关工具,包括html下载器
"""
import json
import time

import requests

from util import logger, commonUtil

Logger = logger.ConsoleLogger(commonUtil.get_file_name(__file__)).getLogger()


class HtmlDownloader(object):
    __type_get = "get"
    __type_post = "post"
    __default_delay = 5

    def __init__(self, request_header=None, request_cookies=None, proxies=None, delay=False,turn_to_json = False):
        """html页面下载器,通过requests模块的request方法进行页面下载,返回response.text

        :param request_header:
        :param request_cookies:
        :param proxies:
        :param delay: 是否设置延迟,可传入boolean或者int类型,为True时默认值为
        :param turn_to_json: 结果是否转换为json
        """
Ejemplo n.º 2
0
 def test_get_file_name(self):
     filepath = "W:/code/git/myProject/pythonStudyRoad/study/dasktop_ui/wx_study.py"
     self.assertEqual("wx_study",
                      commonUtil.get_file_name(filepath, separate="/"))