def CheckHostOnline(host):
try:
ht = Http(host, int(configpara.DEFAULT_TIMEOUT_TIME))
data = ht.head('').data
return True
except Exception, e:
return False
def GetHtml(host):
try:
ht = Http(host, int(configpara.DEFAULT_TIMEOUT_TIME))
ret = ht.get().data
return ret
except Exception, e:
return ""
def GetHtml(host):
try:
ht = Http(host, int(configpara.DEFAULT_TIMEOUT_TIME))
ret = ht.get().data
return ret
except Exception,e:
return ""
def CheckHostOnline(host):
try:
ht = Http(host, int(configpara.DEFAULT_TIMEOUT_TIME))
data = ht.head('').data
return True
except Exception,e:
return False
def Check(self, path):
res = False
path = self._url + path
code = -1
self.SetCode(code)
try:
ht = Http(path, int(self._sp.timeout))
code = ht.head().code
except Exception, e:
self.SetResult(str(e))
self.SaveDb(res)
return res
def Check(self, path):
res = False
path = self._url + path
code = -1
self.SetCode(code)
try:
ht = Http(path, int(self._sp.timeout))
code = ht.head().code
except Exception, e:
self.SetResult(str(e))
self.SaveDb(res)
return res
class Authentication:
http_object = Http()
config = Configuration.global_config['user_credentials']
username = config['username']
password = config['password']
@classmethod
def get_authtoken(cls):
req_body = {
'user': cls.username,
'password': cls.password
}
response = cls.http_object.post('credentials', req_body)
data = response['data']
return data['token']
@classmethod
def delete_authtoken(cls, token):
resource = 'credentials/' + token
cls.http_object.delete(resource)
def __init__(self):
self.url = 'http://oabt007.com/index/index/k/{keyword}/p/{page}'
self.Http = Http()
self.ZMZManager = ZMZManager()
def __init__(self):
self.Http = Http()
self.ZMZManager = ZMZManager()
class zmz_link_json_spider():
def __init__(self):
self.Http = Http()
self.ZMZManager = ZMZManager()
def get_resource_json(self):
pageSize = 20
while True:
resources = self.ZMZManager.get_resource_base(pageSize)
if (resources is None or len(resources) == 0):
break
else:
for res in resources:
# html = self.Http.get_html(res['Url'])
html = self.Http.get_html(res['MApi'])
if html is None or html == '':
print(resourceId, '被屏蔽，后续可重试...')
# 0-未更新json，1-完成解析，2-正在解析json，3-解析json异常，4-被屏蔽拿不到json， 5-404
self.ZMZManager.update_resource_base_status(
res['Id'], 4) # 被屏蔽，可重试
continue
self.get_link(res, html)
secends = random.randint(1, 10)
time.sleep(secends)
if len(resources) < pageSize:
break
# 获取每集基础信息
def get_link(self, data, html):
result = json.loads(html)
if result['status'] == 404:
print('http status: 404, resouceId:', data['Id'])
self.ZMZManager.update_resource_base_status(data['Id'], 5) # 404
return
# pdb.set_trace()
if result['status'] == 1 and 'data' in result and 'info' in result[
'data'] and 'list' in result['data']:
info = result['data']['info']
items = result['data']['list']
# 更新base信息
data['NameCN'] = info.get('cnname', '')
data['NameEN'] = info.get('enname', '')
data['OtherName'] = info.get('aliasname', '')
data['Channel'] = info.get('channel', '')
data['ChannelCN'] = info.get('channel_cn', '')
data['Area'] = info.get('area', '')
data['ShowType'] = info.get('show_type', '')
data['Views'] = info.get('views', 0)
data['Status'] = 2 # 正在解析json
data['UpdateTime'] = datetime.datetime.now()
data['LinkJson'] = json.dumps(items)
print('资源id：', data['Id'], '开始...')
self.get_episode_link(html)
self.ZMZManager.update_resource_base(data)
print('资源id：', data['Id'], '结束...')
# 获取下载链接
def get_episode_link(self, html):
result = json.loads(html)
if 'data' in result and 'info' in result['data'] and 'list' in result[
'data']:
info = result['data']['info']
items = result['data']['list']
for item in items:
season_cn = item['season_cn']
season_num = item['season_num']
for f in item['formats']:
for episode in item['items'][f]:
epi = {}
epi['EpisodeId'] = episode['itemid']
epi['ResourceId'] = info['id']
epi['SeasonCN'] = season_cn
epi['SeasonNum'] = season_num
epi['Formats'] = f
epi['Episode'] = episode['episode']
epi['FileName'] = episode['name']
epi['FileSize'] = episode['size']
try:
# print(epi)
self.ZMZManager.insert_data(epi, 'ResourceEpisode')
except:
print('可能重复插入记录，可忽略...')
pass
if episode['files'] is not None:
for fi in episode['files']:
link = {}
link['ResourceId'] = info['id']
link['EpisodeId'] = episode['itemid']
link['Way'] = fi['way']
link['WayCN'] = fi['way_cn']
link['Url'] = fi['address']
link['Password'] = fi['passwd']
self.ZMZManager.insert_data(
link, 'ResourceEpisodeLink')
def GetServerType(url):
try:
ht = Http(url, int(configpara.DEFAULT_TIMEOUT_TIME))
data = ht.head('').data
except Exception, e:
return ""
def main():
global proxy_client, proxy_server
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
log_filename = 'logs/test_' + time.strftime("%Y%m%d",
time.localtime()) + '.log'
logging.basicConfig(filename=log_filename,
level=logging.INFO,
format=LOG_FORMAT)
# 当前脚本所在目录路径
path = os.path.dirname(os.path.realpath(__file__))
test_config = TestConfig(path)
if len(sys.argv) > 1:
report_name = sys.argv[1]
if 'ls' == sys.argv[1]:
files = getFileName(path + '/config/')
print(Util.pretty(files))
return
else:
report_name = 'default'
# 获取测试用例数据
config = test_config.get_test_case(sys.argv)
# 是否开启代理
is_open_proxy = config.get('BROWSER').get('proxy')
if is_open_proxy:
from browsermobproxy import Server
bmp_path = config.get('BROWSER').get('bmp_path')
logging.info('开启代理 ' + bmp_path)
proxy_server = Server(bmp_path)
proxy_server.start()
proxy_client = proxy_server.create_proxy()
# 浏览器类型
browser_type = config.get('BROWSER').get('type')
# 是否使用H5测试，并指定移动设备名称
h5 = config.get('BROWSER').get('H5', False)
device_name = config.get('BROWSER').get('device_name', 'iPhone 7')
# 是否开启无头模式
headless = config.get('BROWSER').get('headless', False)
if browser_type == 'Firefox':
options = FirefoxOptions()
if headless:
options.add_argument("-headless")
options.page_load_strategy = 'normal'
if h5:
user_agent = "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16"
options.set_preference("general.useragent.override", user_agent)
if is_open_proxy:
options.add_argument('--proxy-server={0}'.format(
proxy_client.proxy))
browser = webdriver.Firefox(options=options)
elif browser_type == 'Chrome':
options = ChromeOptions()
if headless:
options.add_argument("--window-size=1920,1080") # 设置浏览器分辨率（窗口大小）
options.add_argument(
"--start-maximized") # 最大化运行（全屏窗口），不设置，获取元素可能会报错
options.add_argument("--disable-extensions")
options.add_argument('--no-sandbox') # 取消沙盒模式，浏览器的安全性会降低
# 禁用GPU加速，降低资源损耗，only for Windows but not a valid configuration for Linux OS
options.add_argument('--disable-gpu')
options.add_argument('--disable-dev-shm-usage') # 解决资源有限的问题
options.add_argument('--lang=en_US')
options.add_argument("--headless")
options.page_load_strategy = 'normal'
if h5:
mobileEmulation = {'deviceName': device_name}
options.add_experimental_option('mobileEmulation', mobileEmulation)
if is_open_proxy:
options.add_argument('--proxy-server={0}'.format(
proxy_client.proxy))
browser = webdriver.Chrome(options=options)
else:
print('浏览器' + browser_type + ':类型不支持')
return False
logging.info('开始使用 ' + browser_type + ' 浏览器进行自动化测试')
if is_open_proxy:
proxy_client.new_har("req",
options={
'captureHeaders': True,
'captureContent': True
})
if browser_type == 'Firefox' and h5:
browser.set_window_size(360, 640)
else:
browser.maximize_window()
# 浏览器等待时间
# browser.implicitly_wait(10)
url = config.get('WEBSITE').get('url')
browser.get(url)
if is_open_proxy:
Http.logHar(proxy_client.har)
# 执行配置的TEST对象
test = config.get('TEST')
suite = unittest.TestSuite()
m = Menu()
for key in test:
menus = m.getMenuConfig(config, key)
try:
if is_open_proxy:
test_data = [browser, menus, proxy_client]
else:
test_data = [browser, menus]
suite.addTest(
ParametrizedTestCase.parametrize(Action,
'test_menu',
param=test_data))
except AssertExcetion:
print(key + " 断言失败")
# 是否生成报告，默认开启调试模式，不生成报告
debug = config.get('DEBUG', True)
if debug:
runner = unittest.TextTestRunner()
runner.run(suite)
else:
# 报告是否含有截图，DEBUG为False且IMAGE设置为True时生效
image = config.get('IMAGE', False)
report_path = path + '/reports/'
report_file = report_name + "_" + time.strftime(
"%Y%m%d", time.localtime()) + '.html'
fp = open(report_path + report_file, 'wb')
report_title = '你的测试报告'
report_desc = '使用配置:' + report_name + '生成的测试报告'
runner = HTMLTestRunner(stream=fp,
verbosity=2,
images=image,
title=report_title,
description=report_desc,
tester='pystest')
runner.run(suite)
fp.close()
sleep(5)
browser.quit()
# send mail or not
mail = config.get('MAIL')
if not debug and mail and mail.get('SEND'):
email_title = report_title
email_content = report_desc
smtp = Mail(config.get('MAIL'), report_path)
smtp.connect()
smtp.login()
smtp.send(email_title, email_content, report_file)
smtp.quit()
if is_open_proxy:
proxy_client.close()
proxy_server.stop()
def GetServerType(url):
try:
ht = Http(url, int(configpara.DEFAULT_TIMEOUT_TIME))
data = ht.head('').data
except Exception,e:
return ""
def http_object(request):
token = request.cls.token
http_object = Http()
http_object.setHeaders({'Authorization': token})
return http_object
def __init__(self):
self.pc_url = 'http://got001.com/resource.html?code={code}'
self.m_api_url = 'http://m.got001.com/api/v1/static/resource/detail?code={code}'
self.Http = Http()
self.ZMZManager = ZMZManager()
self.WebSelenium = WebSelenium()
class zmz_link_search_spider():
def __init__(self):
self.url = 'http://oabt007.com/index/index/k/{keyword}/p/{page}'
self.Http = Http()
self.ZMZManager = ZMZManager()
def go(self):
pageSize = 20
while True:
resources = self.ZMZManager.get_resource(pageSize)
if (resources is None or len(resources) == 0):
break
else:
for res in resources:
k = []
namecn = res['NameCN'].replace('/', ' ')
nameen = res['NameEN'].replace('/', ' ')
if not namecn or not nameen:
continue
k.append(namecn)
k.append(nameen)
self.get(res['Id'], k)
time.sleep(random.randint(5, 10))
if len(resources) < pageSize:
break
def get(self, resouceId, keyword_array):
result = []
page = 1
while True:
try:
#pdb.set_trace()
url = self.url.format(keyword=parse.quote(
' '.join(keyword_array), encoding='utf-8'),
page=page)
print("当前搜索url：" + url)
html = self.Http.get_html(url)
tree = etree.HTML(html)
link_list = tree.xpath(
'//div[@class="link-list-wrapper"]/ul[@class="link-list"]/li[@data-id]'
)
if link_list and len(link_list) > 0:
for link in link_list:
# pdb.set_trace()
span = link.xpath('./span[@class="name"]')
sr = ResourceLink()
sr['ResourceId'] = resouceId
sr['LinkId'] = int(link.get('data-id', '0'))
sr['Title'] = span[0].text.strip()
sr['MagnetUrl'] = link.get('data-magnet', '')
sr['Ed2kUrl'] = link.get('data-ed2k', '')
if sr['MagnetUrl'].find("magnet:") == -1:
sr['MagnetUrl'] = ''
if sr['Ed2kUrl'].find("ed2k:") == -1:
sr['Ed2kUrl'] = ''
if sr['MagnetUrl'] == '' and sr['Ed2kUrl'] == '':
continue
print(sr)
result.append(sr)
self.insert(sr)
else:
break
page += 1
time.sleep(random.randint(1, 5))
except:
traceback.print_exc() # 打印错误代码行
time.sleep(random.randint(300, 600))
self.ZMZManager.update_resource_status(resouceId, 1)
return result
def insert(self, item):
info = self.ZMZManager.get_one_resource_link(item['ResourceId'],
item['LinkId'])
if info is None:
self.ZMZManager.insert_resource_link(item)
else:
item['Id'] = info['Id']
self.ZMZManager.update_resource_link(item)
def test_menu(self):
self.browser = self.param[0]
self.el = Element(self.param[0])
self.imgs = []
menus = self.param[1]
try:
proxy = self.param[2]
except IndexError:
proxy = {}
# 操作是否成功
assert_result = True
for menu in menus:
print(
time.strftime("%H:%M:%S", time.localtime()) + " " +
menu.get('name'))
# 始终获得当前最后的窗口，所以多要多次使用
search_windows = self.browser.current_window_handle
for handle in self.browser.window_handles:
if handle != search_windows:
self.browser.switch_to.window(handle)
# 是否等待页面消失
if 'wait_disappear' in menu.keys():
wait = WebDriverWait(self.browser, 10, poll_frequency=1)
my_wait = WaitDisappear(menu.get('wait_disappear'))
wait.until(my_wait)
# 切换iframe
if 'iframe' not in menu.keys():
self.browser.switch_to.default_content()
elif 'iframe' in menu.keys() and menu.get('iframe') != 'none':
self.browser.switch_to.frame(menu.get('iframe'))
else:
pass
# 是否等待页面加载
if 'wait' in menu.keys():
wait = WebDriverWait(self.browser, 10, poll_frequency=1)
locator = (menu.get('wait').get('type'),
menu.get('wait').get('content'))
wait.until(EC.presence_of_element_located(locator))
# wait.until(EC.element_to_be_clickable(locator))
# wait.until(EC.visibility_of_element_located(locator))
# http请求监听
if proxy and 'listener' in menu.keys():
proxy.new_har("test",
options={
'captureHeaders': True,
'captureContent': True,
'captureBinaryContent': True
})
self.get_screenshot()
if 'operation' in menu.keys():
for op in menu.get('operation'):
self.operation(op)
self.operation(menu)
# http请求监听
if proxy and 'listener' in menu.keys():
proxy.wait_for_traffic_to_stop(1, 1000)
http = Http()
http.listener(proxy.har, menu.get('listener'))
# 断言
if 'assert' in menu.keys():
try:
self.el.get(
menu.get('assert').get('type'),
menu.get('assert').get('content'), -1)
except NoSuchElementException:
if menu.get('assert').get('assert') == 1:
assert_result = False
# 处理失败，可能会有弹窗，多等一会
sleep(5)
# raise AssertExcetion
else:
if menu.get('assert').get('assert') == 0:
assert_result = False
# raise AssertExcetion
sleep(1)
# 延迟（针对弹框）
# if 'sleep' in menu.keys():
# sleep(menu.get('sleep'))
# 后置操作(关闭弹出页面等)
if 'after_operation' in menu.keys():
for op in menu.get('after_operation'):
try:
self.operation(op)
sleep(1)
except NoSuchElementException:
# 后置操作，找不到元素不做处理
pass
# 断言失败，抛出
if not assert_result:
raise AssertExcetion
def main():
global proxy_client, proxy_server
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
log_filename = 'logs/test_' + time.strftime("%Y%m%d",
time.localtime()) + '.log'
logging.basicConfig(filename=log_filename,
level=logging.INFO,
format=LOG_FORMAT)
# 当前脚本所在目录路径
curpath = os.path.dirname(os.path.realpath(__file__))
# 全局config文件
global_config = {}
global_config_file_path = curpath + "/config.yaml"
if os.path.isfile(global_config_file_path):
gf = open(global_config_file_path, 'r', encoding='utf-8')
global_config = yaml.safe_load(gf.read())
# 是否传入配置文件
if len(sys.argv) > 1:
test_filename = sys.argv[1]
config_file = "/config/" + test_filename + ".yaml"
else:
test_filename = 'default'
config_file = "/config/" + test_filename + '.yaml'
# yaml配置文件是否存在
config_file_path = curpath + config_file
if not os.path.isfile(config_file_path):
print("配置文件不存在 " + config_file_path)
return 1
f = open(config_file_path, 'r', encoding='utf-8')
config = yaml.safe_load(f.read())
# 合并配置
config = Util.recursionMergeTwoDict(global_config, config)
# 是否开启代理
is_open_proxy = config.get('BROWSER').get('proxy')
if is_open_proxy:
from browsermobproxy import Server
bmp_path = config.get('BROWSER').get('bmp_path')
logging.info('开启代理 ' + bmp_path)
proxy_server = Server(bmp_path)
proxy_server.start()
proxy_client = proxy_server.create_proxy()
browser_type = config.get('BROWSER').get('type')
if browser_type == 'Firefox':
options = FirefoxOptions()
options.page_load_strategy = 'normal'
if is_open_proxy:
options.add_argument('--proxy-server={0}'.format(
proxy_client.proxy))
browser = webdriver.Firefox(options=options)
elif browser_type == 'Chrome':
options = ChromeOptions()
options.page_load_strategy = 'normal'
if is_open_proxy:
options.add_argument('--proxy-server={0}'.format(
proxy_client.proxy))
browser = webdriver.Chrome(options=options)
else:
print('浏览器' + browser_type + ':类型不支持')
return False
logging.info('开始使用 ' + browser_type + ' 浏览器进行自动化测试')
if is_open_proxy:
proxy_client.new_har("req",
options={
'captureHeaders': True,
'captureContent': True
})
browser.maximize_window()
# 浏览器等待时间
# browser.implicitly_wait(10)
url = config.get('WEBSITE').get('url')
browser.get(url)
if is_open_proxy:
Http.logHar(proxy_client.har)
# 执行配置的TEST对象
test = config.get('TEST')
suite = unittest.TestSuite()
for key in test:
menus = Menu.getMenuConfig(config, key)
try:
if is_open_proxy:
test_data = [browser, menus, proxy_client]
else:
test_data = [browser, menus]
suite.addTest(
ParametrizedTestCase.parametrize(Action,
'test_menu',
param=test_data))
except AssertExcetion:
print(key + " 断言失败")
report_file_name = 'reports/' + test_filename + "_" + time.strftime(
"%Y%m%d", time.localtime()) + '.html'
fp = open(report_file_name, 'w', encoding='utf-8')
runner = HTMLTestRunner.HTMLTestRunner(stream=fp,
title='你的测试报告',
description='使用配置文件:' +
config_file_path + '生成的测试报告')
runner.run(suite)
fp.close()
sleep(5)
browser.quit()
if is_open_proxy:
proxy_client.close()
proxy_server.stop()