def http_test():
LogManager.build_logger(Constants.LOG_FILE_PATH)
# get request
headers = {'X-Test-Method': 'X-Test-Get'}
http_utils = HttpUtils.get_instance().set_default_headers(headers)
mock_url = 'http://127.0.0.1:17891/index'
headers['Content-Type'] = 'text/plain; charset=utf-8'
query = 'k1=v1&k2=v2'
resp = http_utils.send_http_request(HttpUtils.HTTP_METHOD_GET,
mock_url,
query,
headers=headers,
timeout=0.5)
assert (resp is not None and resp.status_code == 200)
# post request
headers = {'X-Test-Method': 'X-Test-Post'}
http_utils.set_default_headers(headers)
headers['Content-Type'] = 'text/json; charset=utf-8'
data_dict = {'email': '*****@*****.**', 'password': '******'}
# resp = http_utils.send_http_request(
# HttpUtils.HTTP_METHOD_POST_DATA, mock_url, json.dumps(data_dict), headers=headers)
resp = http_utils.send_http_request(HttpUtils.HTTP_METHOD_POST_JSON,
mock_url,
data_dict,
headers=headers)
assert (resp is not None and resp.status_code == 200)
LogManager.clear_log_handles()
def init_logger():
logdir_path = LoadConfigs.get_testenv_configs().get('logdir_path')
logdir_path = logdir_path.replace('{project}', project_path)
logfile_path = os.path.join(
logdir_path,
'pytest_log_%s.txt' % SysUtils.get_current_date_and_time())
LogManager.build_logger(logfile_path)
def py_base_ex03():
sys.path.append(os.getenv('PYPATH'))
from utils import Constants
from utils import LogManager
from utils import SysUtils
try:
logger = LogManager.build_logger(Constants.LOG_FILE_PATH)
logger.info('import external modules test.')
utils = SysUtils().get_instance()
utils.run_sys_cmd('python --version')
finally:
LogManager.clear_log_handles()
def __init__(self):
self.__logger = LogManager.get_logger()
self.__sysutils = SysUtils.get_instance()
self.__configs = LoadConfigs.get_email_configs()
self.__msg = MIMEMultipart('mixed')
self.__content_text = 'Default mail template from API test.'
def __init__(self, report_root_path):
'''
Constructor
'''
self.__logger = LogManager.get_logger()
self.__report_root_path = report_root_path
self.__handtest_dir_path = os.path.join(self.__report_root_path,
'handTest')
self.sysutils = SysUtils()
def __init__(self, itest_collect_interval):
'''
Constructor
'''
self.__is_stopped = True
self.__logger = LogManager.get_logger()
self.__sys_utils = SysUtils()
self.__adb_utils = AdbUtils()
# note: this value should be greater than interval config in iTest
self.__wait_time_between_check = itest_collect_interval + 1
def __test_clearup_main(self):
# the adb connection maybe disconnect when running the monkey
if self.__adbutils.is_devices_connected():
self.__profile_monitor.stop_monitor()
self.__adbutils.stop_app(self.__test_pkg_name)
self.__filter_shell_logcat_exception()
self.__filter_shell_logcat_anr()
self.__pull_all_testing_logs()
self.__create_monkey_test_report()
if self.__is_profile_test_ok():
time.sleep(1)
self.__profile_monitor.pull_itest_logfiles(
self.__log_dir_path_for_win)
self.__chart_parser.build_all_profile_charts()
else:
self.__logger.error('Device disconnect!')
LogManager.clear_log_handles()
if Constants.IS_CREATE_ARCHIVE:
self.__create_archive_report_file()
def py_parallel_demo24(is_thread=False):
import logging
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures import wait
sys.path.append(os.getenv('PYPATH'))
from utils import Constants, LogManager
LogManager.build_logger(Constants.LOG_FILE_PATH,
stream_log_level=logging.DEBUG)
logger = LogManager.get_logger()
base_path = os.path.join(os.getenv('HOME'), 'Downloads/tmp_files')
file_paths = [os.path.join(base_path, 'test_data.csv')]
file_paths.append(os.path.join(base_path, 'test_log.txt'))
file_paths.append(os.path.join(base_path, 'not_exist.txt'))
workers_num = 3
executor = ThreadPoolExecutor(max_workers=workers_num) \
if is_thread else ProcessPoolExecutor(max_workers=workers_num)
f_submit = [
executor.submit(checkfile_process, logger, path) for path in file_paths
]
f_done = wait(f_submit, return_when='ALL_COMPLETED')
for future in f_done.done:
if future.exception() is None:
if future.result() is not None:
# if error, result() will directly raise Exception
logger.debug('result: ' + future.result())
else:
logger.error('exception ===> ' + str(future.exception()))
logger.debug('[%d] main process done.' % os.getpid())
LogManager.clear_log_handles()
def __init__(self, test_pkg_name, run_mins):
'''
Constructor
'''
self.__test_pkg_name = test_pkg_name
self.__run_mins = int(run_mins)
cur_time = SysUtils.get_current_date_and_time()
self.__log_root_path = os.path.join(os.getcwd(), 'monkeyreports')
self.__log_dir_path_for_win = os.path.join(self.__log_root_path,
cur_time)
self.__log_dir_path_for_shell = '/data/local/tmp/monkey_test_logs'
self.__exec_log_path = os.path.join(self.__log_dir_path_for_win,
'run_log.log')
self.__device_props_file_path = os.path.join(
self.__log_dir_path_for_win, 'device_props.log')
self.__app_dump_file_path = os.path.join(self.__log_dir_path_for_win,
'app_info.log')
self.__monkey_log_path_for_shell = '%s/%s' % (
self.__log_dir_path_for_shell, 'monkey_log.log')
self.__logcat_log_path_for_shell = '%s/%s' % (
self.__log_dir_path_for_shell, 'logcat_full_log.log')
self.__logcat_exception_file_name = 'logcat_exception.log'
self.__logcat_exception_path_for_shell = '%s/%s' % (
self.__log_dir_path_for_shell, self.__logcat_exception_file_name)
self.__logcat_anr_file_name = 'logcat_anr.log'
self.__logcat_anr_path_for_shell = '%s/%s' % (
self.__log_dir_path_for_shell, self.__logcat_anr_file_name)
SysUtils.create_dir(self.__log_dir_path_for_win)
self.__logger = LogManager.build_logger(self.__exec_log_path)
self.__sysutils = SysUtils()
self.__adbutils = AdbUtils()
self.__monitor = MonkeyMonitor()
self.__profile_monitor = ProfileMonitor(
Constants.ITEST_COLLECT_INTERVAL)
self.__chart_parser = ChartParser(self.__log_dir_path_for_win)
self.__report = MonkeyReport(self.__log_dir_path_for_win,
self.__logcat_exception_file_name,
self.__logcat_anr_file_name)
def setup_test_session(request):
'''
Setup for test session:
1) init logger and allure; 2) load configs and testcases.
'''
load_configs()
init_logger()
load_testcases()
init_allure_env()
logger = LogManager.get_logger()
logger.info(
'[session setup]: 1) init logger and allure; 2) load configs and testcases.'
)
def clear():
logger.info('[session clearup] done.')
LogManager.clear_log_handles()
request.addfinalizer(clear)
if not os.path.exists(output_dir_path):
self.__logger.error('test output dir is not exist: ' +
output_dir_path)
return ''
if not os.path.isdir(output_dir_path):
self.__logger.error('invalid test out dir: ' + output_dir_path)
return ''
return output_dir_path
if __name__ == '__main__':
isTest = True
if isTest:
print('Done')
else:
# init logger and configs
LogManager.build_logger(Constants.LOG_FILE_PATH)
cfg_file_path = os.path.join(os.getenv('PYPATH'), 'apitest',
'configs.ini')
LoadConfigs.load_configs(cfg_file_path)
mail_content = 'API test done.\nPlease read details of results in attachment.'
mail = EmailHelper.get_intance()
mail.set_content_text(mail_content).send_email()
LogManager.clear_log_handles()
print('email helper test DONE.')
# @Version : Python 3.8.5
import json
import time
from pathlib import Path
from threading import RLock
from urllib.parse import urlencode
import execjs
from utils import RequestManager, LogManager, ciphers
from hunter.videos import ParseError, CookieError, DownloadType, Provider
from hunter.videos.tencent_video import get_cvid, VideoType
from privacy import get_unid
from hunter.videos.tencent_video.login.mock import QQLogin
logger = LogManager('video_tencent_video_get_download_url').file()
CUR_DIR = Path(__file__).parent
with open(CUR_DIR / 'tencent.js', 'r', encoding='utf-8') as frt:
EXECJS_TENCENT = execjs.compile(frt.read())
class DownloadURL(object):
_lock = RLock()
_cookie_file = CUR_DIR.parent / 'cookie.txt'
def __init__(self,
play_url: str,
platform: str = '10201',
app_ver: str = '3.5.57',
sdtfrom: str = 'v1010'):
self._cookie = None
# -*- coding: utf-8 -*-
# @Author : BrotherBe
# @Time : 2020/9/9 8:18
# @Version : Python 3.8.5
import json
import re
import time
from contextlib import suppress
from public import BackEndPort
from utils import LogManager, MongoDBManager, RequestManager
from hunter.videos import VideoDetailData, VideoCategory, BaseVideo, Code, DownloadStatus
from hunter.videos.tencent_video import get_cvid
logger = LogManager('video_tencent_video_service').file()
RE_columnInfo = re.compile(r'var COLUMN_INFO = (.*)')
RE_coverInfo = re.compile(r'var COVER_INFO = (.*)')
RE_videoINFO = re.compile(r'var VIDEO_INFO = (.*)')
RE_vid = re.compile(r'vid=(\w+)')
video_col = MongoDBManager().video.get_collection('tencent_video')
# 创建索引
video_col.create_index('video_id', unique=True, background=True)
class Details(object):
def __init__(self, play_url):
self.play_url = play_url
logger.info(self.play_url)
self.headers = {
def __init__(self):
'''
Constructor
'''
self.__logger = LogManager.get_logger()
self.__adbutils = AdbUtils()
def __init__(self):
self.__logger = LogManager.get_logger()
self.__headers = {}
def __init__(self):
self.__logger = LogManager.get_logger()
self.__xlsx = XlsxUtils.get_instance()
self.__test_cases = []
self.__header = []
def __init__(self):
self.__logger = LogManager.get_logger()
self.__sys_utils = SysUtils.get_instance()
from utils import (
check_columns,
plot_dist_of_cols,
check_nan_value,
correct_column_type,
remove_cont_cols_with_small_std,
remove_cont_cols_with_unique_value,
get_time_diff,
LogManager,
timer,
)
# global setting
LogManager.created_filename = os.path.join(conf.LOG_DIR,
'feature_engineering.log')
logger = LogManager.get_logger(__name__)
# global varirable
POWER_ON_HOURS_CATE_LIST = [-9999, 452.75, 905.5, 1358.25, 9999.
] # [-1.811, 452.75 , 905.5 , 1358.25 , 1811.]
BINS_FOR_CUT_POWER_ON_HOURS_FEAT = 4
DEFAULT_MISSING_FLOAT = -1.234
STD_THRESHOLD_FOR_REMOVING_COLUMNS = 1
FAULT_LABEL = 1
USING_LABEL = 'tag'
DROP_UNIQUE_COL_THRESHOLD = 1 # i.e. the unique number of this column should be larger than threshold
DROP_NAN_COL_THRESHOLD = 30 # i.e. 30%
NORMALIZED_SMART_COLS = [
'smart_1_normalized', 'smart_3_normalized', 'smart_7_normalized',
'smart_9_normalized', 'smart_187_normalized', 'smart_189_normalized',
'smart_193_normalized', 'smart_191_normalized', 'smart_195_normalized'
def setup_class(cls):
cls.__logger = LogManager.get_logger()
cls.__http_utils = HttpUtils.get_instance()
# -*- coding: utf-8 -*-
# @Author : BrotherBe
# @Time : 2020/9/14 0:52
# @Version : Python 3.8.5
import time
from public import RabbitMQQueue
from utils import RabbitMQConsumer, MongoDBManager, m3u8_download, remove_special_characters, LogManager, mp4_download
from hunter.videos import DownloadStatus, DownloadType, get_storage_path, get_provider, Provider
from hunter.videos.tencent_video.tencent_video_service import TencentVideoService
video_db = MongoDBManager().video
logger = LogManager('video_download').file()
def _video_download(provider, filter, download_type, params):
logger.info(f'开始下载: {provider=}, {filter=}, {download_type=}')
col = video_db.get_collection(provider)
col.update_one(
filter=filter,
update={"$set": {
'download.status': DownloadStatus.downloading,
'download.reason': '',
'download.time': time.strftime('%Y-%m-%d %H:%M:%S'),
}}
)
record = col.find_one(filter)
filename = f"{remove_special_characters(record['name'])}_{record['video_id']}.{download_type}"
filepath = get_storage_path(provider) / filename
logger.info(f'下载路径: {filepath}')
is_success = False
def __init__(self):
self.__logger = LogManager.get_logger()
self.__sheet = None
# -*- coding: utf-8 -*-
# @Author : BrotherBe
# @Time : 2020/10/2 10:54
# @Version : Python 3.8.5
from flask import Flask, request, render_template
from public import RabbitMQQueue, FrontEndPort
from utils import LogManager, RabbitMQProducer
app = Flask(__name__)
logger = LogManager('videos_flask').file()
producer = RabbitMQProducer(RabbitMQQueue.video_download_name, use_filter=True)
@app.route('/videos/add', methods=['GET'])
def add_form():
return '''
<body bgcolor="DodgerBlue">
<div style="text-align:center">
<font></font><br/><font></font><br/><font></font><br/>
<font face="宋体" size="+5" color="#F0F8FF">添加影片链接</font><br/>
<font></font><br/>
<font></font><br/>
<form action ='/videos/add' method='post'>
<input type="txt" rows="2" cols="60" placeholder="请输入影片链接...."
name='play_url' style="height:40px;width:500px;">
<button type="submit" class="search-submit"
style="height:38px;width:60px;">添加</button>
</form>
</div>
'''
def clear():
logger.info('[session clearup] done.')
LogManager.clear_log_handles()
# -*- coding: utf-8 -*-
# @Author : BrotherBe
# @Time : 2020/9/9 0:10
# @Version : Python 3.8.5
from flask import Flask, request, jsonify
from public import BackEndPort
from utils import LogManager
from hunter.videos import Code, VideoException
from hunter.videos.tencent_video.decrypt.get_download_url import DownloadURL
app = Flask('video_api_server')
logger = LogManager('video_api_server').file()
@app.route("/video/tencent_video/get_download_url", methods=["POST"])
def get_download_url():
"""
@api {get} /video/tencent_video/get_download_url 获取腾讯视频下载地址
@apiName get_download_url
@apiVersion 0.0.1
@apiGroup tencent_video api
@apiParam {String} play_url 视频播放地址
@apiSuccessExample {json} 返回示例:
{
"code": 10000,
"data": {
"play_url": "https://v.qq.com/x/cover/5y07fkhxzj48wcj/g0022cy9bdp.html",
"download_type": "m3u8",
"download": [
{
def __init__(self):
self._logger = LogManager.get_logger()
from contextlib import suppress
from pathlib import Path
from threading import Lock
from selenium import webdriver
from selenium.common import exceptions
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait
from utils import LogManager, RequestManager
from config_account import account
from hunter.videos.tencent_video.login.captcha import Captcha
logger = LogManager('tencent_video_login').file()
class QQLogin(object):
_lock = Lock()
_executable_path = r'/usr/bin/chromedriver'
image_path = Path(__file__).parent / 'tencent_video_captcha'
image_path.mkdir(parents=True, exist_ok=True)
def __init__(self,
username: str = account.Hunter.Videos.TencentVideo.username,
password: str = account.Hunter.Videos.TencentVideo.password,
timeout: float = 60):
self._username = username
self._password = password
self.index_url = 'https://graph.qq.com/oauth2.0/show?redirect_uri=https%3A%2F%2Faccess.video.qq.com%2Fuser%2Fauth_login%3Fvappid%3D11059694%26vsecret%3Dfdf61a6be0aad57132bc5cdf78ac30145b6cd2c1470b0cfe%26raw%3D1%26type%3Dqq%26appid%3D101483052&which=Login&display=pc&response_type=code&client_id=101483052'
ret_ok = resp.json()['results']
expected_ok = json.loads(case[self.CASE_SCHEMA_EXP_MSG])['results']
assert (ret_ok == expected_ok)
@pytest.allure.severity(pytest.allure.severity_level.CRITICAL)
def test_index_post_01(self):
case = LoadCases.get_instance().get_tc_data_dict(self.__cur_case)
headers = LoadCases.format_headers_to_dict(
case[self.CASE_SCHEMA_HEADER])
resp = self.__http_utils.send_http_request(
case[self.CASE_SCHEMA_METHOD],
case[self.CASE_SCHEMA_URL],
case[self.CASE_SCHEMA_BODY],
headers=headers)
# assert(resp is not None and resp.status_code == int(case['RetCode']))
self.base_http_assert(resp)
if __name__ == '__main__':
LogManager.build_logger(Constants.LOG_FILE_PATH,
stream_log_level=logging.DEBUG)
file_path = os.path.join(os.path.dirname(os.getcwd()), 'TestCases.xlsx')
LoadCases.get_instance().pre_load_sheet(
file_path, 'Module01').load_all_cases_by_sheet()
pytest.main(['-v', '-s', 'test_module_01.py'])
LogManager.clear_log_handles()
print('test module 01 DONE.')
def __init__(self):
'''
Constructor
'''
self.__logger = LogManager.get_logger()
load_testcases()
init_allure_env()
logger = LogManager.get_logger()
logger.info(
'[session setup]: 1) init logger and allure; 2) load configs and testcases.'
)
def clear():
logger.info('[session clearup] done.')
LogManager.clear_log_handles()
request.addfinalizer(clear)
if __name__ == '__main__':
isTest = True
if isTest:
print(sys.path)
print('Done')
else:
# load_configs()
init_logger()
load_testcases()
tcs = LoadCases.get_instance().get_loaded_tcs()
LogManager.get_logger().info(tcs)
LogManager.get_logger().info('conftest DONE.')
LogManager.clear_log_handles()
