def cmp_urls(url_name, expect_url, fact_url):
list_params = ["p", "mp", "q", "csp", "r"]
expect_url_dict = get_url_params_dict(expect_url, url_name)
fact_url_dict = get_url_params_dict(fact_url, url_name)
for param in list_params:
write_log(param.center(20, "-"))
expect_param_dict = expect_url_dict.get(param)
fact_param_dict = fact_url_dict.get(param)
cmp_dict_data_pass_diff(expect_param_dict, fact_param_dict)
def cmp_url():
url_expect = raw_input("input the expect url: ").strip()
url_fact = raw_input("input the expect url: ").strip()
url_dict_expect = get_url_params_dict(url_expect, "url")
url_dict_fact = get_url_params_dict(url_fact, "url")
c_list_param = ["p", "mp", "q", "csp", "r"]
for c_param in c_list_param:
write_log(c_param.center(50, "="))
cmp_dict_data_pass_diff(url_dict_expect.get(c_param),
url_dict_fact.get(c_param))
def get_log_new(file_path, key_name):
log_file = get_new_file(file_path)
if not log_file:
return {}
log_value = read_txt_to_list(file_path + log_file)
if not log_value:
return {}
list_values = log_value[-1].split("\t")
list_keys = DICT_LOG_KEYS.get(key_name)
if len(list_values) != len(list_keys):
write_log("the length of log key:%s is diff to vlue: %s" %
(len(list_keys), len(list_values)))
return dict(zip(list_keys, list_values))
def get_response(self,
request_url,
method="GET",
headers={},
data={},
ex_code=200):
# write_log(u"请求的method: {0}".format(method))
# if headers:
# write_log(u"请求的headers: {0}".format(headers))
# if data:
# write_log(u"请求的data: {0}".format(data))
try:
if method.upper() == 'POST':
if headers.get('Content-type') == "application/json":
data = json.dumps(data)
write_log(u"请求的url: {0}".format(request_url))
self.res = requests.post(request_url,
headers=headers,
data=data,
timeout=1)
else:
request_url += urllib.urlencode(data)
write_log(u"请求的url: {0}".format(request_url))
self.res = requests.get(request_url,
headers=headers,
timeout=10)
if self.res.status_code != ex_code:
write_log(
u'返回的状态码错误，预期是：{0}，实际是：{1}'.format(ex_code,
self.res.status_code),
'error')
except Exception as e:
write_log(u"请求失败，原因：{0}".format(e), 'error')
return self.res
def get_log_info(server_type, log_type="", check_time=""):
dict_log_time = get_log_time(check_time)
dict_replace = {}
if log_type:
dict_replace.setdefault('{log_type}', log_type)
dict_replace.update(**dict_log_time)
if not DICT_LOG.get(server_type):
write_log(
'the server_type {0} you choose is not exists, please check you config!!!'
.format(server_type), 'warn')
return "", "", "", ""
root_path = DICT_LOG.get(server_type).get('root_path')
log_path = DICT_LOG.get(server_type).get('log_path')
log_key = DICT_LOG.get(server_type).get('log_key')
for k, v in dict_replace.items():
log_path = log_path.replace(k, v)
return root_path, log_path, log_key
def get_url_params_dict(url, url_name):
# type: (object, object) -> object
"""将url转化为dict格式
:rtype: object
"""
dict_params = {}
try:
url_path, url_params = url.split("?")
dict_params['url_path'] = url_path
list_kv = url_params.split("&")
for kv in list_kv:
k, v = kv.split("=")
if k in DICT_PARAM_KEYS.keys():
v = get_url_dict_params(k, v, url_name)
dict_params.setdefault(k, v)
except StandardError, e:
print(e)
write_log("{0} is not correct: {1}".format(url_name, url), 'warn')
def get_log(server_type, log_type, via_type, via_info):
"""
获取单个接口末尾行日志，以map形式输出
:param server_type: 服务端类型,如1-adn_net，2-midway_server，3-midway_tracking
:param log_type: 接口类型，如request，impression，click，only_impression,install等
:param via_type: 匹配日志的方式 1-通过关键字 2-通过行号
:param via_info: via_type的1则via_info为 via_type=2则为行号
:return: 匹配的接口日志行信息
"""
root_path, log_path, log_key = get_log_info(server_type, log_type)
if not root_path:
# write_log("get log path failed, server_type: {0}, log_type: {1}".format(server_type, log_type))
return {}
if int(via_type) == 1:
list_via_info = via_info.split("+")
cmd = 'cat {0}'.format(log_path)
for each_via_info in list_via_info:
cmd += "| grep {1}".format(each_via_info)
try:
log_lines = [
_[:-1] for _ in os.popen(cmd).readlines() if len(_) > 0
]
if len(log_lines) < 1:
raise None
list_param_value = log_lines[0].split("\t")
except StandardError:
write_log(
"get log failed , not exists or not unique, via {0}".format(
cmd), 'warn')
return {}
else:
try:
log_lines = [
_[:-1] for _ in read_txt_to_list(log_path) if len(_) > 0
]
if len(log_lines) < int(via_info):
raise None
list_param_value = log_lines[int(via_info)].split("\t")
except StandardError, e:
write_log(
"get log failed , file_path: {0}, msg: {1}".format(
log_path, e), 'warn')
return {}
def get_url_dict_params(param_name, param_value, url_name):
"""将特殊字段转化为dict格式
:rtype: object
"""
if not param_value:
write_log(
"{1}: the value param {0} is empty".format(param_name, url_name),
'warn')
return {param_name: param_value}
param_k = DICT_PARAM_KEYS.get(param_name)
if param_name in ("mp", "q", "csp"):
param_v = decode_q_mp(param_value)
else:
param_v = decode_p(param_value)
param_v = param_v.split("|")
if len(param_v) < 2:
write_log(
"build {0} faild:\n{2}: {1}".format(param_name, param_v, url_name),
'error')
return {param_name: param_value}
elif len(param_k) != len(param_v):
write_log(
"{3}: build {0}, the size of key {1} is diff to size of value {2}".
format(param_name, len(param_k), len(param_v), url_name), 'warn')
return dict(zip(param_k, param_v))
def cmp_log(key_expect="", key_fact="", list_log_type="all", via_line=False):
if list_log_type == "all":
list_log_type = DICT_LOG_SEARCH.keys()
for log_type in list_log_type:
log_action = DICT_LOG_SEARCH.get(log_type)
list_log_type = DICT_LOG_SEARCH.keys()
if not log_action:
write_log(
"The log_type {0} is not in the config, please check!".format(
log_type), 'warn')
continue
write_log('start to cmp_log {0}'.format(log_type), 'info')
if via_line:
# 如果不是查询所有的，那么以为着没有获取到广告，则不用
log_action["expect"][3] = -2
log_action["expect"][2] = 2
log_action["fact"][3] = -1
log_action["fact"][2] = 2
else:
log_action["expect"][3] += key_expect
log_action["fact"][3] += key_fact
log_expct = apply(get_log, tuple(log_action.get("expect")))
log_fact = apply(get_log, tuple(log_action.get("fact")))
cmp_dict_data_pass_diff(log_expct, log_fact, log_action.get("pass"))
def run(action_type):
if action_type == "1":
url = raw_input("input the url: ").strip()
d = get_url_params_dict(url, "url")
c_list_param = ["p", "mp", "q", "csp", "r"]
for c_param in c_list_param:
write_log(c_param.center(20, "-"))
c_value = d.get(c_param)
if c_value and isinstance(c_value, dict):
c_value = json.dumps(c_value, indent=4)
write_log(c_value)
else:
param_name = raw_input(
"p | q | mp | csp |r, choose param name: ").strip()
param_value = raw_input("input param value: ").strip()
c_value = get_url_dict_params(param_name, param_value, "test")
if c_value and isinstance(c_value, dict):
c_value = json.dumps(c_value, indent=4)
write_log(c_value)
def get_log_new(file_path, key_name):
log_file = get_new_file(file_path)
if not log_file:
write_log(
"The file folder is empty or no file updated in 1 minute: %s" %
file_path, "warn")
return {}, []
log_file = log_file[:-1]
print("The full log path is: %s" % log_file, "info")
log_value = read_txt_to_list(log_file)
if not log_value:
write_log("The file is empty: %s" % log_file, "warn")
return {}, []
list_values = log_value[-1].split("\t")
list_keys = DICT_LOG_KEYS.get(key_name)
if len(list_values) != len(list_keys):
write_log(
"the length of log key:%s is diff to vlue: %s" %
(len(list_keys), len(list_values)), "warn")
return dict(zip(list_keys, list_values)), list_keys
fact_param_dict = fact_url_dict.get(param)
cmp_dict_data_pass_diff(expect_param_dict, fact_param_dict)
if __name__ == "__main__":
import json
# mv
url = 'http://{domain}/openapi/ad/v3?api_version=0.9&tnum=1&orientation=1&package_name=com.vstudio.camera360&install_ids=%5B177536538%5D&ad_type=94&ad_source_id=1&country_code=CN&timezone=GMT%252B09%253A00&mnc=50&ping_mode=1&category=0&native_info=%5B%7B"id"%3A2%2C"ad_num"%3A60%7D%5D&platform=2&cache2=51297.835938&unit_id=11542&os_version=11.1.2&sdk_version=MI_3.3.2&ttc_ids=%5B177536538%5D&req_type=1&exclude_ids=%5B177536538%5D&cache1=63937.039062&http_req=2&idfv=2603D94D-7465-4388-B0C3-BDFA40DCBE32&ad_num=60&app_version_name=8.3.4&screen_size=375.000000x812.000000&mcc=440&app_id=+90044&display_cids=%5B177536538%5D&idfa=C98B99BB-1B2F-434E-BE53-FD651B03FB6A&offset=0&only_impression=1&power_rate=80&sign=ad65e4be9a35160da6481391cff361fb&keyword=4MElRoztHFV0RaElinjso0vlRrJQYrxQh0RlG0veWvElRozMD%252B30RaElinRso0vlRrf2hbxXYZRlG0v0D%252Bf3HrdrwDXE8PV0WvElRozghdi0RaEl4MElRovlRretJoRlG0v0inv%2FinDAfAiBR0MpRovlRozsYrh0RaElRUDMWUjFiAhAi0RpRoSKogT%253D&language=en-JP&ui_orientation=26&version_flag=1&openidfa=74C87BBD-C675-EE2A-7651-6EBAC72835EA&ats=4MElRozGVTcsY7KbhTcBDrQThrcB4VeXDkxAR0v1RdxBJkVp6N%253D%253D&useragent=Mozilla%2F5.0%2520%28iPhone%253B%2520CPU%2520iPhone%2520OS%252010_2_1%2520like%2520Mac%2520OS%2520X%29%2520AppleWebKit%2F602.4.6%2520%28KHTML%252C%2520like%2520Gecko%29%2520Mobile%2F14D27%2520Camera360%2F8.3.4&model=iPhone8%252C1&sub_ip=192.168.6.233&network_type=9'
# 3rd
url = "http://{domain}/openapi/ad/v3?tnum=1&orientation=1&package_name=com.vstudio.camera360&install_ids=%5B177536538%5D&ad_type=94&ad_source_id=1&country_code=CN&timezone=GMT%252B09%253A00&mnc=50&ping_mode=1&category=0&native_info=%5b%7b%22id%22%3a2%2c%22ad_num%22%3a60%7d%5d&platform=2&cache2=51297.835938&unit_id=1341&os_version=11.1.2&sdk_version=MI_3.3.2&ttc_ids=%5B177536538%5D&req_type=1&exclude_ids=%5B177536538%5D&cache1=63937.039062&api_version=1.3&http_req=2&idfv=2603D94D-7465-4388-B0C3-BDFA40DCBE32&ad_num=60&app_version_name=8.3.4&screen_size=375.000000x812.000000&mcc=440&app_id=25259&display_cids=%5B177536538%5D&idfa=C98B99BB-1B2F-434E-BE53-FD651B03FB6A&offset=0&only_impression=1&power_rate=80&sign=34b5ffd843857e456693bd9e4fa52f62&keyword=4MElRoztHFV0RaElinjso0vlRrJQYrxQh0RlG0veWvElRozMD%252B30RaElinRso0vlRrf2hbxXYZRlG0v0D%252Bf3HrdrwDXE8PV0WvElRozghdi0RaEl4MElRovlRretJoRlG0v0inv%2FinDAfAiBR0MpRovlRozsYrh0RaElRUDMWUjFiAhAi0RpRoSKogT%253D&language=en-JP&ui_orientation=26&version_flag=1&openidfa=74C87BBD-C675-EE2A-7651-6EBAC72835EA&ats=4MElRozGVTcsY7KbhTcBDrQThrcB4VeXDkxAR0v1RdxBJkVp6N%253D%253D&useragent=Mozilla%2F5.0%2520%28iPhone%253B%2520CPU%2520iPhone%2520OS%252010_2_1%2520like%2520Mac%2520OS%2520X%29%2520AppleWebKit%2F602.4.6%2520%28KHTML%252C%2520like%2520Gecko%29%2520Mobile%2F14D27%2520Camera360%2F8.3.4&model=iPhone8%252C1&sub_ip=192.168.6.233&network_type=9"
expect_domain = "test-net.rayjump.com"
fact_domain = "test-adnet.rayjump.com"
list_checks = [
"only_impression_url", "impression_url", "notice_url", "click_url"
]
write_log(url.format(domain=expect_domain), "info")
write_log(url.format(domain=fact_domain), "info")
expect_res = requests.get(url.format(domain=expect_domain)).json()
# write_log json.dumps(expect_res, indent=4)
fact_res = requests.get(url.format(domain=fact_domain)).json()
# write_log json.dumps(fact_res, indent=4)
if expect_res.get("status") == fact_res.get("status") and fact_res.get(
"status") == 1:
list_expect_campaign = expect_res.get("data").get("ads")
list_fact_campaign = fact_res.get("data").get("ads")
set_expect_ids = set([_.get("id") for _ in list_expect_campaign])
set_fact_ids = set([_.get("id") for _ in list_fact_campaign])
list_campaign_id = list(set_expect_ids.intersection(set_fact_ids))
if list_campaign_id:
campaign_id = list_campaign_id[0]
expect_campaign = [
try:
log_lines = [
_[:-1] for _ in read_txt_to_list(log_path) if len(_) > 0
]
if len(log_lines) < int(via_info):
raise None
list_param_value = log_lines[int(via_info)].split("\t")
except StandardError, e:
write_log(
"get log failed , file_path: {0}, msg: {1}".format(
log_path, e), 'warn')
return {}
list_param_name = DICT_LOG_KEYS.get(log_key)
if len(list_param_value) != len(list_param_name):
write_log(
'param size is wrong,\n\texpect is {0}; fact is {1}\n\tplease check {2}!!!'
.format(len(list_param_name), len(list_param_value),
DICT_LOG.get(server_type).get('log_params_path')), 'warn')
return dict(zip(list_param_name, list_param_value))
def cmp_log(key_expect="", key_fact="", list_log_type="all", via_line=False):
if list_log_type == "all":
list_log_type = DICT_LOG_SEARCH.keys()
for log_type in list_log_type:
log_action = DICT_LOG_SEARCH.get(log_type)
list_log_type = DICT_LOG_SEARCH.keys()
if not log_action:
write_log(
"The log_type {0} is not in the config, please check!".format(
log_type), 'warn')