def sole_file_data(instance):
"""对拦截curl记录保存文件, 做处理去掉重复行.
:param instance: DataStore实例.
"""
lines = instance.open_file_data()
if lines:
sole_file = '%s_distinct%s' % os.path.splitext(
instance.get_report_file()) # 去重后的记录文件
if os.path.exists(sole_file):
os.remove(sole_file) # 删除之前生成文件
log.debug('Original: %s' % len(lines))
sole_data = set(lines)
num_date = 0
with open(sole_file, 'ab') as rf:
log_time = "#### %s\n" % str(datetime.datetime.now())
rf.write(str(log_time))
for _line_ in sole_data:
if _line_.startswith('####'): # 取消原来日期分组
num_date += 1
continue
rf.write(_line_)
log.debug('Nowadays: %s' % (len(sole_data) - num_date))
print('All is Ok')
else:
print('Data is None')
def sole_file_data(instance):
"""对拦截curl记录保存文件, 做处理去掉重复行.
:param instance: DataStore实例.
"""
lines = instance.open_file_data()
if lines:
sole_file = '%s_distinct%s' % os.path.splitext(
instance.get_report_file()) # 去重后的记录文件
if os.path.exists(sole_file):
os.remove(sole_file) # 删除之前生成文件
log.debug('Original: %s' % len(lines))
sole_data = set(lines)
num_date = 0
with open(sole_file, 'ab') as rf:
log_time = "#### %s\n" % str(datetime.datetime.now())
rf.write(str(log_time))
for _line_ in sole_data:
if _line_.startswith('####'): # 取消原来日期分组
num_date += 1
continue
rf.write(_line_)
log.debug('Nowadays: %s' % (len(sole_data) - num_date))
print('All is Ok')
else:
print('Data is None')
def main():
"""提供外部 entry points 而用.
"""
parser = OptionParser()
parser.add_option("-f",
"--file",
type="string",
dest="file",
help="store the curl script data file.")
parser.add_option("-n",
"--num",
type="int",
dest="num",
default=None,
help="execute the script line number specified.")
parser.add_option("-c",
"--count",
type="int",
dest="count",
default=0,
help="perform the following line count.")
(options, args) = parser.parse_args()
log.debug("options:%s\n" % options)
if not options.file:
log.error("specified file required parameters are missing.")
return
file_name = options.file
_curl_script = ScriptExecute(file_name, report_bool=True, lazy_bone=None)
if options.num is None:
_curl_script.run_script_total()
else:
line_num = options.num
line_count = options.count
_curl_script.run_script_lines(line_num, line_count)
def __loop_line(self, num):
"""运行行列表里指定的行.
:param num: 行编号, 正整数.
"""
log_name = self.__path_result_file() % num
if num > 0:
num -= 1 # 文档行标, 实例索引起点不一
else:
raise Exception("num参数必须是正整数.")
line = self.script_lines[num].strip().replace("\n", "")
# log.debug(line.startswith('curl'))
if line.startswith('curl'):
if self.lazy_bone:
line = self.lazy_bone.process_regular(line)
log.debug(line)
log.debug(line.split()[-1])
if self.report_bool:
os.system('%s > %s' % (line, log_name))
else:
os.system('%s' % line)
log.debug('-*' * 50)
def __loop_line(self, num):
"""运行行列表里指定的行.
:param num: 行编号, 正整数.
"""
log_name = self.__path_result_file() % num
if num > 0:
num -= 1 # 文档行标, 实例索引起点不一
else:
raise Exception("num参数必须是正整数.")
line = self.script_lines[num].strip().replace("\n", "")
# log.debug(line.startswith('curl'))
if line.startswith('curl'):
if self.lazy_bone:
line = self.lazy_bone.process_regular(line)
log.debug(line)
log.debug(line.split()[-1])
if self.report_bool:
os.system('%s > %s' % (line, log_name))
else:
os.system('%s' % line)
log.debug('-*' * 50)
def main():
"""提供外部 entry points 而用.
"""
parser = OptionParser()
parser.add_option("-f", "--file",
type="string",
dest="file",
help="store the curl script data file.")
parser.add_option("-n", "--num",
type="int",
dest="num",
default=None,
help="execute the script line number specified.")
parser.add_option("-c", "--count",
type="int",
dest="count",
default=0,
help="perform the following line count.")
(options, args) = parser.parse_args()
log.debug("options:%s\n" % options)
if not options.file:
log.error("specified file required parameters are missing.")
return
file_name = options.file
_curl_script = ScriptExecute(file_name, report_bool=True, lazy_bone=None)
if options.num is None:
_curl_script.run_script_total()
else:
line_num = options.num
line_count = options.count
_curl_script.run_script_lines(line_num, line_count)
def request_process(request, frame='django'):
"""输出并记录 request post(dict) 传入值.
捕获和验证参数以供 curl 再使用, 验证从 curl 传来值.
"""
req_method = request.method
if frame == 'django':
is_secure = request.is_secure()
req_dict = request.POST if req_method == "POST" else request.GET
get_host = request.get_host()
get_full_path = request.get_full_path()
elif frame == 'tornado':
is_secure = request.protocol == 'https'
req_dict = request.arguments
get_host = request.host
get_full_path = request.path
elif frame == 'pyramid':
is_secure = request.scheme == 'https'
req_dict = request.POST if req_method == "POST" else request.GET
get_host = request.host
get_full_path = request.path
else:
is_secure = False
req_dict = {}
get_host = ''
get_full_path = ''
str_post = ''
protocol = 'http://' if not is_secure else 'https://'
request_url = '%s%s%s' % (protocol, get_host, get_full_path)
# 输出传入值开始
log.debug('URL: %s' % request_url)
if len(req_dict) > 0:
log.debug(req_dict)
for (key, value) in req_dict.items():
str_post = '&'.join((str_post, '%s=%s' % (key, value)))
str_post = str_post.strip('&')
log.debug('Data String: %s' % str_post)
log.debug('-*' * 50)
if req_method == "POST" and post_data_saved and str_post:
# 记录传入值
line = cur_instance.hold_data_require(
request, request_url=request_url, data=str_post, frame=frame)
cur_instance.save_line_data(line)
return get_full_path
def returned_wrapper(request, *args, **kwargs):
try:
# 查看并控制台核实传入数据
full_path = request_process(request, frame)
# 计算后端程序执行时间
if exec_time_print:
with Timer() as t:
response = func(request, *args, **kwargs)
log.debug("%s => %s ms" % (full_path, t.millisecond))
line = "%-25s %s => %s ms\n" % (
full_path, 8 * ' ', t.millisecond)
time_instance.save_line_data(line)
else:
response = func(request, *args, **kwargs)
return response
except Exception as e:
# 异常时保存下数据
cur_instance.save_file_data()
log.exception(e)
traceback.print_exc(file=sys.stdout)