Example #1
0
def CheckHostOnline(host):
    try:
        ht = Http(host, int(configpara.DEFAULT_TIMEOUT_TIME))
        data = ht.head('').data
        return True
    except Exception, e:
        return False
Example #2
0
def GetHtml(host):
    try:
        ht = Http(host, int(configpara.DEFAULT_TIMEOUT_TIME))
        ret = ht.get().data
        return ret
    except Exception, e:
        return ""
Example #3
0
def GetHtml(host):
    try:
        ht = Http(host, int(configpara.DEFAULT_TIMEOUT_TIME))
        ret = ht.get().data
        return ret
    except Exception,e:
        return ""
Example #4
0
def CheckHostOnline(host):
    try:
        ht = Http(host, int(configpara.DEFAULT_TIMEOUT_TIME))
        data = ht.head('').data
        return True
    except Exception,e:
        return False
Example #5
0
    def Check(self, path):
        res = False
        path = self._url + path
        code = -1
        self.SetCode(code)

        try:
            ht = Http(path, int(self._sp.timeout))
            code = ht.head().code
        except Exception, e:
            self.SetResult(str(e))
            self.SaveDb(res)
            return res
Example #6
0
    def Check(self, path):
        res = False
        path = self._url + path
        code = -1
        self.SetCode(code)

        try:
            ht = Http(path, int(self._sp.timeout))
            code = ht.head().code
        except Exception, e:
            self.SetResult(str(e))
            self.SaveDb(res)
            return res
Example #7
0
class Authentication:
    http_object = Http()
    config = Configuration.global_config['user_credentials']
    username = config['username']
    password = config['password']

    @classmethod
    def get_authtoken(cls):
        req_body = {
          'user': cls.username,
          'password': cls.password
        }
        response = cls.http_object.post('credentials', req_body)
        data = response['data']
        return data['token']

    @classmethod
    def delete_authtoken(cls, token):
        resource = 'credentials/' + token
        cls.http_object.delete(resource)
Example #8
0
 def __init__(self):
     self.url = 'http://oabt007.com/index/index/k/{keyword}/p/{page}'
     self.Http = Http()
     self.ZMZManager = ZMZManager()
Example #9
0
 def __init__(self):
     self.Http = Http()
     self.ZMZManager = ZMZManager()
Example #10
0
class zmz_link_json_spider():
    def __init__(self):
        self.Http = Http()
        self.ZMZManager = ZMZManager()

    def get_resource_json(self):
        pageSize = 20
        while True:
            resources = self.ZMZManager.get_resource_base(pageSize)
            if (resources is None or len(resources) == 0):
                break
            else:
                for res in resources:
                    # html = self.Http.get_html(res['Url'])
                    html = self.Http.get_html(res['MApi'])
                    if html is None or html == '':
                        print(resourceId, '被屏蔽,后续可重试...')
                        # 0-未更新json,1-完成解析,2-正在解析json,3-解析json异常,4-被屏蔽拿不到json, 5-404
                        self.ZMZManager.update_resource_base_status(
                            res['Id'], 4)  # 被屏蔽,可重试
                        continue
                    self.get_link(res, html)
                    secends = random.randint(1, 10)
                    time.sleep(secends)
            if len(resources) < pageSize:
                break

    # 获取每集基础信息
    def get_link(self, data, html):
        result = json.loads(html)
        if result['status'] == 404:
            print('http status: 404, resouceId:', data['Id'])
            self.ZMZManager.update_resource_base_status(data['Id'], 5)  # 404
            return
        # pdb.set_trace()
        if result['status'] == 1 and 'data' in result and 'info' in result[
                'data'] and 'list' in result['data']:
            info = result['data']['info']
            items = result['data']['list']
            # 更新base信息
            data['NameCN'] = info.get('cnname', '')
            data['NameEN'] = info.get('enname', '')
            data['OtherName'] = info.get('aliasname', '')
            data['Channel'] = info.get('channel', '')
            data['ChannelCN'] = info.get('channel_cn', '')
            data['Area'] = info.get('area', '')
            data['ShowType'] = info.get('show_type', '')
            data['Views'] = info.get('views', 0)
            data['Status'] = 2  # 正在解析json
            data['UpdateTime'] = datetime.datetime.now()
            data['LinkJson'] = json.dumps(items)
            print('资源id:', data['Id'], '开始...')
            self.get_episode_link(html)
            self.ZMZManager.update_resource_base(data)
            print('资源id:', data['Id'], '结束...')

    # 获取下载链接
    def get_episode_link(self, html):
        result = json.loads(html)
        if 'data' in result and 'info' in result['data'] and 'list' in result[
                'data']:
            info = result['data']['info']
            items = result['data']['list']
            for item in items:
                season_cn = item['season_cn']
                season_num = item['season_num']
                for f in item['formats']:
                    for episode in item['items'][f]:
                        epi = {}
                        epi['EpisodeId'] = episode['itemid']
                        epi['ResourceId'] = info['id']
                        epi['SeasonCN'] = season_cn
                        epi['SeasonNum'] = season_num
                        epi['Formats'] = f
                        epi['Episode'] = episode['episode']
                        epi['FileName'] = episode['name']
                        epi['FileSize'] = episode['size']
                        try:
                            # print(epi)
                            self.ZMZManager.insert_data(epi, 'ResourceEpisode')
                        except:
                            print('可能重复插入记录,可忽略...')
                            pass
                        if episode['files'] is not None:
                            for fi in episode['files']:
                                link = {}
                                link['ResourceId'] = info['id']
                                link['EpisodeId'] = episode['itemid']
                                link['Way'] = fi['way']
                                link['WayCN'] = fi['way_cn']
                                link['Url'] = fi['address']
                                link['Password'] = fi['passwd']
                                self.ZMZManager.insert_data(
                                    link, 'ResourceEpisodeLink')
Example #11
0
def GetServerType(url):
    try:
        ht = Http(url, int(configpara.DEFAULT_TIMEOUT_TIME))
        data = ht.head('').data
    except Exception, e:
        return ""
Example #12
0
def main():
    global proxy_client, proxy_server
    LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
    log_filename = 'logs/test_' + time.strftime("%Y%m%d",
                                                time.localtime()) + '.log'
    logging.basicConfig(filename=log_filename,
                        level=logging.INFO,
                        format=LOG_FORMAT)

    # 当前脚本所在目录路径
    path = os.path.dirname(os.path.realpath(__file__))

    test_config = TestConfig(path)

    if len(sys.argv) > 1:
        report_name = sys.argv[1]
        if 'ls' == sys.argv[1]:
            files = getFileName(path + '/config/')
            print(Util.pretty(files))
            return
    else:
        report_name = 'default'

    # 获取测试用例数据
    config = test_config.get_test_case(sys.argv)

    # 是否开启代理
    is_open_proxy = config.get('BROWSER').get('proxy')
    if is_open_proxy:
        from browsermobproxy import Server
        bmp_path = config.get('BROWSER').get('bmp_path')
        logging.info('开启代理 ' + bmp_path)
        proxy_server = Server(bmp_path)
        proxy_server.start()
        proxy_client = proxy_server.create_proxy()

    # 浏览器类型
    browser_type = config.get('BROWSER').get('type')

    # 是否使用H5测试,并指定移动设备名称
    h5 = config.get('BROWSER').get('H5', False)
    device_name = config.get('BROWSER').get('device_name', 'iPhone 7')

    # 是否开启无头模式
    headless = config.get('BROWSER').get('headless', False)

    if browser_type == 'Firefox':
        options = FirefoxOptions()
        if headless:
            options.add_argument("-headless")
        options.page_load_strategy = 'normal'
        if h5:
            user_agent = "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16"
            options.set_preference("general.useragent.override", user_agent)

        if is_open_proxy:
            options.add_argument('--proxy-server={0}'.format(
                proxy_client.proxy))
        browser = webdriver.Firefox(options=options)
    elif browser_type == 'Chrome':
        options = ChromeOptions()
        if headless:
            options.add_argument("--window-size=1920,1080")  # 设置浏览器分辨率(窗口大小)
            options.add_argument(
                "--start-maximized")  # 最大化运行(全屏窗口),不设置,获取元素可能会报错
            options.add_argument("--disable-extensions")
            options.add_argument('--no-sandbox')  # 取消沙盒模式,浏览器的安全性会降低
            # 禁用GPU加速,降低资源损耗,only for Windows but not a valid configuration for Linux OS
            options.add_argument('--disable-gpu')
            options.add_argument('--disable-dev-shm-usage')  # 解决资源有限的问题
            options.add_argument('--lang=en_US')
            options.add_argument("--headless")
        options.page_load_strategy = 'normal'
        if h5:
            mobileEmulation = {'deviceName': device_name}
            options.add_experimental_option('mobileEmulation', mobileEmulation)

        if is_open_proxy:
            options.add_argument('--proxy-server={0}'.format(
                proxy_client.proxy))
        browser = webdriver.Chrome(options=options)
    else:
        print('浏览器' + browser_type + ':类型不支持')
        return False

    logging.info('开始使用 ' + browser_type + ' 浏览器进行自动化测试')

    if is_open_proxy:
        proxy_client.new_har("req",
                             options={
                                 'captureHeaders': True,
                                 'captureContent': True
                             })

    if browser_type == 'Firefox' and h5:
        browser.set_window_size(360, 640)
    else:
        browser.maximize_window()
    # 浏览器等待时间
    # browser.implicitly_wait(10)

    url = config.get('WEBSITE').get('url')
    browser.get(url)
    if is_open_proxy:
        Http.logHar(proxy_client.har)

    # 执行配置的TEST对象
    test = config.get('TEST')
    suite = unittest.TestSuite()
    m = Menu()
    for key in test:
        menus = m.getMenuConfig(config, key)
        try:
            if is_open_proxy:
                test_data = [browser, menus, proxy_client]
            else:
                test_data = [browser, menus]
            suite.addTest(
                ParametrizedTestCase.parametrize(Action,
                                                 'test_menu',
                                                 param=test_data))
        except AssertExcetion:
            print(key + " 断言失败")

    # 是否生成报告,默认开启调试模式,不生成报告
    debug = config.get('DEBUG', True)
    if debug:
        runner = unittest.TextTestRunner()
        runner.run(suite)
    else:
        # 报告是否含有截图,DEBUG为False且IMAGE设置为True时生效
        image = config.get('IMAGE', False)
        report_path = path + '/reports/'
        report_file = report_name + "_" + time.strftime(
            "%Y%m%d", time.localtime()) + '.html'
        fp = open(report_path + report_file, 'wb')
        report_title = '你的测试报告'
        report_desc = '使用配置:' + report_name + '生成的测试报告'
        runner = HTMLTestRunner(stream=fp,
                                verbosity=2,
                                images=image,
                                title=report_title,
                                description=report_desc,
                                tester='pystest')
        runner.run(suite)
        fp.close()

    sleep(5)
    browser.quit()

    # send mail or not
    mail = config.get('MAIL')
    if not debug and mail and mail.get('SEND'):
        email_title = report_title
        email_content = report_desc
        smtp = Mail(config.get('MAIL'), report_path)
        smtp.connect()
        smtp.login()
        smtp.send(email_title, email_content, report_file)
        smtp.quit()

    if is_open_proxy:
        proxy_client.close()
        proxy_server.stop()
Example #13
0
def GetServerType(url):
    try:
        ht = Http(url, int(configpara.DEFAULT_TIMEOUT_TIME))
        data = ht.head('').data
    except Exception,e:
        return ""
Example #14
0
def http_object(request):
    token = request.cls.token
    http_object = Http()
    http_object.setHeaders({'Authorization': token})
    return http_object
Example #15
0
 def __init__(self):
     self.pc_url = 'http://got001.com/resource.html?code={code}'
     self.m_api_url = 'http://m.got001.com/api/v1/static/resource/detail?code={code}'
     self.Http = Http()
     self.ZMZManager = ZMZManager()
     self.WebSelenium = WebSelenium()
Example #16
0
class zmz_link_search_spider():
    def __init__(self):
        self.url = 'http://oabt007.com/index/index/k/{keyword}/p/{page}'
        self.Http = Http()
        self.ZMZManager = ZMZManager()

    def go(self):
        pageSize = 20
        while True:
            resources = self.ZMZManager.get_resource(pageSize)
            if (resources is None or len(resources) == 0):
                break
            else:
                for res in resources:
                    k = []
                    namecn = res['NameCN'].replace('/', ' ')
                    nameen = res['NameEN'].replace('/', ' ')
                    if not namecn or not nameen:
                        continue
                    k.append(namecn)
                    k.append(nameen)
                    self.get(res['Id'], k)
                    time.sleep(random.randint(5, 10))
            if len(resources) < pageSize:
                break

    def get(self, resouceId, keyword_array):
        result = []
        page = 1
        while True:
            try:
                #pdb.set_trace()
                url = self.url.format(keyword=parse.quote(
                    ' '.join(keyword_array), encoding='utf-8'),
                                      page=page)
                print("当前搜索url:" + url)
                html = self.Http.get_html(url)
                tree = etree.HTML(html)
                link_list = tree.xpath(
                    '//div[@class="link-list-wrapper"]/ul[@class="link-list"]/li[@data-id]'
                )
                if link_list and len(link_list) > 0:
                    for link in link_list:
                        # pdb.set_trace()
                        span = link.xpath('./span[@class="name"]')
                        sr = ResourceLink()
                        sr['ResourceId'] = resouceId
                        sr['LinkId'] = int(link.get('data-id', '0'))
                        sr['Title'] = span[0].text.strip()
                        sr['MagnetUrl'] = link.get('data-magnet', '')
                        sr['Ed2kUrl'] = link.get('data-ed2k', '')
                        if sr['MagnetUrl'].find("magnet:") == -1:
                            sr['MagnetUrl'] = ''
                        if sr['Ed2kUrl'].find("ed2k:") == -1:
                            sr['Ed2kUrl'] = ''
                        if sr['MagnetUrl'] == '' and sr['Ed2kUrl'] == '':
                            continue
                        print(sr)
                        result.append(sr)
                        self.insert(sr)
                else:
                    break
                page += 1
                time.sleep(random.randint(1, 5))
            except:
                traceback.print_exc()  # 打印错误代码行
                time.sleep(random.randint(300, 600))
        self.ZMZManager.update_resource_status(resouceId, 1)
        return result

    def insert(self, item):
        info = self.ZMZManager.get_one_resource_link(item['ResourceId'],
                                                     item['LinkId'])
        if info is None:
            self.ZMZManager.insert_resource_link(item)
        else:
            item['Id'] = info['Id']
            self.ZMZManager.update_resource_link(item)
Example #17
0
    def test_menu(self):
        self.browser = self.param[0]
        self.el = Element(self.param[0])
        self.imgs = []
        menus = self.param[1]
        try:
            proxy = self.param[2]
        except IndexError:
            proxy = {}

        # 操作是否成功
        assert_result = True

        for menu in menus:
            print(
                time.strftime("%H:%M:%S", time.localtime()) + " " +
                menu.get('name'))

            # 始终获得当前最后的窗口,所以多要多次使用
            search_windows = self.browser.current_window_handle
            for handle in self.browser.window_handles:
                if handle != search_windows:
                    self.browser.switch_to.window(handle)

            # 是否等待页面消失
            if 'wait_disappear' in menu.keys():
                wait = WebDriverWait(self.browser, 10, poll_frequency=1)
                my_wait = WaitDisappear(menu.get('wait_disappear'))
                wait.until(my_wait)

            # 切换iframe
            if 'iframe' not in menu.keys():
                self.browser.switch_to.default_content()
            elif 'iframe' in menu.keys() and menu.get('iframe') != 'none':
                self.browser.switch_to.frame(menu.get('iframe'))
            else:
                pass

            # 是否等待页面加载
            if 'wait' in menu.keys():
                wait = WebDriverWait(self.browser, 10, poll_frequency=1)
                locator = (menu.get('wait').get('type'),
                           menu.get('wait').get('content'))
                wait.until(EC.presence_of_element_located(locator))
                # wait.until(EC.element_to_be_clickable(locator))
                # wait.until(EC.visibility_of_element_located(locator))

            # http请求监听
            if proxy and 'listener' in menu.keys():
                proxy.new_har("test",
                              options={
                                  'captureHeaders': True,
                                  'captureContent': True,
                                  'captureBinaryContent': True
                              })

            self.get_screenshot()
            if 'operation' in menu.keys():
                for op in menu.get('operation'):
                    self.operation(op)

            self.operation(menu)

            # http请求监听
            if proxy and 'listener' in menu.keys():
                proxy.wait_for_traffic_to_stop(1, 1000)
                http = Http()
                http.listener(proxy.har, menu.get('listener'))

            # 断言
            if 'assert' in menu.keys():
                try:
                    self.el.get(
                        menu.get('assert').get('type'),
                        menu.get('assert').get('content'), -1)
                except NoSuchElementException:
                    if menu.get('assert').get('assert') == 1:
                        assert_result = False
                        # 处理失败,可能会有弹窗,多等一会
                        sleep(5)
                        # raise AssertExcetion
                else:
                    if menu.get('assert').get('assert') == 0:
                        assert_result = False
                        # raise AssertExcetion
                sleep(1)

            # 延迟(针对弹框)
            # if 'sleep' in menu.keys():
            #     sleep(menu.get('sleep'))

            # 后置操作(关闭弹出页面等)
            if 'after_operation' in menu.keys():
                for op in menu.get('after_operation'):
                    try:
                        self.operation(op)
                        sleep(1)
                    except NoSuchElementException:
                        # 后置操作,找不到元素不做处理
                        pass

            # 断言失败,抛出
            if not assert_result:
                raise AssertExcetion
Example #18
0
def main():
    global proxy_client, proxy_server
    LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
    log_filename = 'logs/test_' + time.strftime("%Y%m%d",
                                                time.localtime()) + '.log'
    logging.basicConfig(filename=log_filename,
                        level=logging.INFO,
                        format=LOG_FORMAT)

    # 当前脚本所在目录路径
    curpath = os.path.dirname(os.path.realpath(__file__))

    # 全局config文件
    global_config = {}
    global_config_file_path = curpath + "/config.yaml"
    if os.path.isfile(global_config_file_path):
        gf = open(global_config_file_path, 'r', encoding='utf-8')
        global_config = yaml.safe_load(gf.read())

    # 是否传入配置文件
    if len(sys.argv) > 1:
        test_filename = sys.argv[1]
        config_file = "/config/" + test_filename + ".yaml"
    else:
        test_filename = 'default'
        config_file = "/config/" + test_filename + '.yaml'

    # yaml配置文件是否存在
    config_file_path = curpath + config_file
    if not os.path.isfile(config_file_path):
        print("配置文件不存在 " + config_file_path)
        return 1

    f = open(config_file_path, 'r', encoding='utf-8')
    config = yaml.safe_load(f.read())

    # 合并配置
    config = Util.recursionMergeTwoDict(global_config, config)

    # 是否开启代理
    is_open_proxy = config.get('BROWSER').get('proxy')
    if is_open_proxy:
        from browsermobproxy import Server
        bmp_path = config.get('BROWSER').get('bmp_path')
        logging.info('开启代理 ' + bmp_path)
        proxy_server = Server(bmp_path)
        proxy_server.start()
        proxy_client = proxy_server.create_proxy()

    browser_type = config.get('BROWSER').get('type')
    if browser_type == 'Firefox':
        options = FirefoxOptions()
        options.page_load_strategy = 'normal'
        if is_open_proxy:
            options.add_argument('--proxy-server={0}'.format(
                proxy_client.proxy))
        browser = webdriver.Firefox(options=options)
    elif browser_type == 'Chrome':
        options = ChromeOptions()
        options.page_load_strategy = 'normal'
        if is_open_proxy:
            options.add_argument('--proxy-server={0}'.format(
                proxy_client.proxy))
        browser = webdriver.Chrome(options=options)
    else:
        print('浏览器' + browser_type + ':类型不支持')
        return False

    logging.info('开始使用 ' + browser_type + ' 浏览器进行自动化测试')

    if is_open_proxy:
        proxy_client.new_har("req",
                             options={
                                 'captureHeaders': True,
                                 'captureContent': True
                             })

    browser.maximize_window()
    # 浏览器等待时间
    # browser.implicitly_wait(10)

    url = config.get('WEBSITE').get('url')
    browser.get(url)
    if is_open_proxy:
        Http.logHar(proxy_client.har)

    # 执行配置的TEST对象
    test = config.get('TEST')
    suite = unittest.TestSuite()
    for key in test:
        menus = Menu.getMenuConfig(config, key)
        try:
            if is_open_proxy:
                test_data = [browser, menus, proxy_client]
            else:
                test_data = [browser, menus]
            suite.addTest(
                ParametrizedTestCase.parametrize(Action,
                                                 'test_menu',
                                                 param=test_data))
        except AssertExcetion:
            print(key + " 断言失败")

    report_file_name = 'reports/' + test_filename + "_" + time.strftime(
        "%Y%m%d", time.localtime()) + '.html'
    fp = open(report_file_name, 'w', encoding='utf-8')
    runner = HTMLTestRunner.HTMLTestRunner(stream=fp,
                                           title='你的测试报告',
                                           description='使用配置文件:' +
                                           config_file_path + '生成的测试报告')
    runner.run(suite)
    fp.close()

    sleep(5)
    browser.quit()

    if is_open_proxy:
        proxy_client.close()
        proxy_server.stop()