def __enter__(self):
logger.debug("Start class OutputData!")
self.content_format = self.args.op_format.upper()
self.output_type = self.args.op_type.upper()
if self.output_type == "FILE":
self.fp = open(self.args.op_file, "w")
self.csv_list = []
elif self.output_type == "KAFKA":
# check topic exists
self.topic = self.args.op_topic
kafka_topic = NewTopic(name=self.topic,
num_partitions=1,
replication_factor=1)
client = KafkaAdminClient(bootstrap_servers=self.args.op_bootstrap)
try:
client.delete_topics([kafka_topic])
client.create_topics([kafka_topic])
except KafkaError:
logger.warn(
"delete or create kafka topic raised error, ignore it!")
self.producer = KafkaProducer(
bootstrap_servers=self.args.op_bootstrap)
elif self.output_type == "ES" or self.output_type == "ElasticSearch".upper(
):
self.es = Elasticsearch(
hosts=self.args.op_es_hosts,
sniff_on_start=True,
# sniff_on_connection_fail=True,
sniffer_timeout=20,
# http_auth=('user', 'secret')
)
self.es_index = self.args.op_index
return self
def __init__(self, name=None):
# Handle data file absolute name
if name:
# If provided file name
self.name = name
else:
try:
# Read data file from config
self.name = config.get_conf("file").get("case_file")
except ConfKeyNotFound:
# Use default file path
self.name = os.path.abspath("./data/cmdb_data.xlsx")
if not os.path.exists(self.name):
message = "Data file not exist!"
raise FileNotFoundError2(message)
# Read contents from data file
self.workbook = load_workbook(filename=self.name)
self.sheets = self.workbook.sheetnames
# Only use the 1st sheet.
if len(self.sheets) > 1:
logger.warn("The excel file have more than one sheet, now we use the first sheet.")
self.content = self.workbook[self.sheets[0]]
self.case_content = {}
self.ids_content = {}
def match_result(fragment, response):
"""
(
fragment (
(entity.situation[*], min),
(1.2)
),
(2.1),
(3.1)
)
:param fragment:
:param response:
:return:
"""
result = []
if not fragment:
return [result]
for frag in fragment:
logger.info("In match_result: Frag is {}, type is {}".format(frag, type(frag))) # frag: (entity.situation[*], min)
pattern = parse(frag[0])
logger.info("response.json is {}, type is {}".format(response.json(), type(response.json())))
match_list = [match.value for match in pattern.find(response.json())]
logger.info("match_list is {}".format(match_list))
try:
d = {}
string = "result = {}({})".format(frag[1], match_list)
exec(string, d)
result.append(str(d["result"]))
except IndexError:
logger.warn("Dependency_fragment: {} not specify function name for match pattern.".format(frag))
else:
result.append(str(match_list).strip("[|]|(|)|{|}"))
return [result]
def wrapper(*args, **kwargs):
t = now()
result = func(*args, **kwargs)
ms = take_ms(t)
if ms > 500:
logger.warn('Execute [{}] takes {}ms'.format(func.__name__, ms))
return result
def wrapper(*args, **kw):
try:
return func(*args, **kw)
except Exception as e:
db.session.rollback()
if isinstance(e, ServiceException):
if e.error_code < ErrorCode.INTERNAL_ERROR:
logger.warn(e.get_log_msg())
else:
logger.fatal(e.get_log_msg())
return jsonify({
'code': e.error_code,
'msg': e.msg,
'data': None
})
else:
exstr = traceback.format_exc()
logger.fatal(str(e) + u'\n详情：' + exstr)
return jsonify({
'code': ErrorCode.INTERNAL_ERROR,
'msg': exstr,
'data': None
})
finally:
db.session.close()
def run(self, debug):
try:
# selenium_util.debug_info()
# 获取开始运行时间
start_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
if self.object_type == '页面元素':
logger.info('正在查找页面元素选择器')
temp_result = self.query_element_selectors()
if temp_result[0]:
selector1, selector2 = temp_result[1]
for run_times in range(0, self.run_times + 1):
logger.info('正在运行第%s次' % str(run_times + 1))
for try_times in range(0, self.try_for_failure + 1):
temp_result = self.operate_page_element(
selector1, selector2, self.exec_operation,
self.input_params)
if temp_result[0]:
logger.info('运行成功')
result = [True, '成功', '']
break
else:
logger.warn(
'运行失败：%s，正在进行第%s次重试' %
(temp_result[1], str(try_times + 1)))
result = [temp_result[0], '失败', temp_result[1]]
else:
result = [False, '失败', temp_result[1]]
elif self.object_type == '系统函数':
result = self.exec_system_func(self.op_object,
self.input_params)
if not result[0]:
logger.error('执行出错，原因：%s' % result[2])
elif self.object_type == '数据库':
# logger.info('步骤操作对象为数据库')
result = [False, '失败', '暂时不支持']
assert_result = self.exec_assert()
if result[0] and assert_result[0]:
result = [True, '成功', '']
else:
if not result[0]:
result = [False, '失败', '步骤运行失败：' + result[2]]
elif assert_result[0]:
result = [False, '失败', '断言失败：' + assert_result[1]]
except Exception as e:
result = [False, '失败', '%s' % e]
finally:
if not debug:
logger.info(
'======================正在记录用例步骤运行结果到测试报告-用例步骤执行明细表======================'
)
data = (self.execution_num, self.plan_id, self.case_id,
self.step_id, self.order, self.page_name,
self.op_object, self.exec_operation, self.input_params,
self.output_params, self.assert_type,
self.assert_pattern, self.run_times,
self.try_for_failure, result[1], result[2], start_time,
0)
test_reporter.insert_report_for_case_step(data)
return result
def __run(username, password):
"""
运行函数
:param username: 登录用户名
:param password: 登录密码
:return:
"""
try:
token, user_slag = login(username, password)
except Exception as e:
logger.warn('Username=%s登录异常，详情' % username, str(e))
return
if token is None:
logger.warn('Username=%s登录失败' % username)
return
info = get_info(token, user_slag)
info['username'] = username
info['password'] = base64.b64encode(password.encode(encoding='utf-8'))
try:
update_info(info)
except Exception as ex:
logger.error(ex)
session.rollback()
logger.info('账号%s，Leetcode今日签到成功' % username)
def assert_element_exists(self, selenium_util, element_selector):
result = selenium_util.find_element_by_locator_adapter(
element_selector)
if result[0]: #如果找到了
logger.info('找到元素：%s' % result[1])
else:
logger.warn('未找到元素：%s' % result[1])
return result
def _on_job_error(self, event):
from service.job_service import JobService
e = event.exception
job_id = event.job_id
logger.warn('定时任务%s发生错误，将被移除调度器' % job_id)
logger.error(str(e))
task = JobService.get_job(job_id)
self.remove_job(task)
def predict(self, one_sample):
if self.is_leaf:
return self.obj
else:
fea_value = one_sample[self.split_fea_idx]
match_child = self.__find_child(fea_value)
if match_child is None:
logger.warn("没找到相应的子节点，用当前节点的值替代预测值")
return self.obj
return match_child.predict(one_sample)
def add_user(who, hashes: list):
logger.debug('{} has {} chunks to register'.format(who.username,
len(hashes)))
for h in hashes:
if h == '':
logger.warn('{} sent invalid chunk hash'.format(who.username))
continue
h = Hash(h)
register_chunk(h, who)
keep_chunk_alive(who, h)
def add_user(who, hashes: list):
logger.debug('{} has {} chunks to register'.
format(who.username, len(hashes)))
for h in hashes:
if h == '':
logger.warn('{} sent invalid chunk hash'.format(who.username))
continue
h = Hash(h)
register_chunk(h, who)
keep_chunk_alive(who, h)
def host_chunk(frm, chk):
chk = check_for_string(chk)
user = chunk.find_user_for_storing(chk)
if user is None:
logger.warn("""could not find any user to store {} (from {})."""
"""Chunk currently has {} hosted instances"""
.format(chk.pretty(), frm.username, chunk.get_redundancy(chk)))
return
send_chunk_to(user, chk)
def host_chunk(frm, chk):
chk = check_for_string(chk)
user = chunk.find_user_for_storing(chk)
if user is None:
logger.warn("""could not find any user to store {} (from {})."""
"""Chunk currently has {} hosted instances""".format(
chk.pretty(), frm.username, chunk.get_redundancy(chk)))
return
send_chunk_to(user, chk)
def send_chunk_to(client: hash.Hash, chk):
chk = check_for_string(chk)
from_cli = chunk.get_chunk_owner(chk)
if from_cli is None:
logger.warn('could not find any user hosting {}'.format(chk.pretty()))
return
logger.debug('{} is being sent from {} to {}'.format(
chk.pretty(), from_cli.username, client.username))
protocol.send_CSND(from_cli, client, 1, chk)
protocol.send_CSND(client, from_cli, 0, chk)
chunk.register_chunk(chk, client)
def send_chunk_to(client: hash.Hash, chk):
chk = check_for_string(chk)
from_cli = chunk.get_chunk_owner(chk)
if from_cli is None:
logger.warn('could not find any user hosting {}'.format(chk.pretty()))
return
logger.debug('{} is being sent from {} to {}'
.format(chk.pretty(), from_cli.username, client.username))
protocol.send_CSND(from_cli, client, 1, chk)
protocol.send_CSND(client, from_cli, 0, chk)
chunk.register_chunk(chk, client)
def find_element(self, selector1, selector2):
# 优先使用selector1查找
result = selenium_util.find_element_by_locator_adapter(selector1)
if result[0]: #如果找到了
logger.info('找到元素：%s' % result[1])
return result
else:
logger.info('用选择器1:%s未找到元素，开始用选择器2:%s查找' % (selector1, selector2))
result = selenium_util.find_element_by_locator_adapter(selector2)
if result[0]: #如果找到了
logger.info('找到元素：%s' % result[1])
else:
logger.warn('未找到元素：%s' % result[1])
return result
def loop(client_port, username):
thread = threading.Thread(target=listen, args=(client_port,))
thread.daemon = True
thread.start()
loop = asyncio.get_event_loop()
coro = loop.create_connection(lambda: Client(loop, client_port, username), server_addr, server_port)
try:
loop.run_until_complete(coro)
loop.run_forever()
except KeyboardInterrupt:
logger.warn('keyboard interrupt')
pass
loop.close()
def loop(client_port, username):
thread = threading.Thread(target=listen, args=(client_port, ))
thread.daemon = True
thread.start()
loop = asyncio.get_event_loop()
coro = loop.create_connection(lambda: Client(loop, client_port, username),
server_addr, server_port)
try:
loop.run_until_complete(coro)
loop.run_forever()
except KeyboardInterrupt:
logger.warn('keyboard interrupt')
pass
loop.close()
def test(self, url, content):
"""
根据配置（反扒响应条件）检查是否是期望的内容（如果URL被反扒，响应内容返回错误的信息）
检查通过，则URL内容正确；检查失败，说明该请求被反扒.
:param url:
:param content:
:return:
"""
ret = True
failure_list = self.get_failure_str_list(url)
if failure_list:
for failure_item in failure_list:
if failure_item in content:
logger.warn('Wrong response content, fail condition: %s', failure_item)
ret = False
break
return ret
def listen(port):
eloop = asyncio.new_event_loop()
asyncio.set_event_loop(eloop)
coro = eloop.create_server(Server, '0.0.0.0', port)
server = eloop.run_until_complete(coro)
logger.info('listening on {}'.format(server.sockets[0].getsockname()))
try:
eloop.run_forever()
except KeyboardInterrupt:
logger.warn('keyboard interrupt')
pass
server.close()
eloop.run_until_complete(server.wait_closed())
eloop.close()
def listen(port):
eloop = asyncio.new_event_loop()
asyncio.set_event_loop(eloop)
coro = eloop.create_server(Server, '0.0.0.0', port)
server = eloop.run_until_complete(coro)
logger.info('listening on {}'.format(server.sockets[0].getsockname()))
try:
eloop.run_forever()
except KeyboardInterrupt:
logger.warn('keyboard interrupt')
pass
server.close()
eloop.run_until_complete(server.wait_closed())
eloop.close()
def run(self, debug):
try:
if_plan_run_fail = False
plan_id_list_for_failure = [] # 存放执行失败、未执行的测试计划的ID
logger.info('======================正在同步更新与待运行测试计划关联的所有用例树节点信息======================')
result = self.sync_case_tree_node_info_for_testplans()
if not result[0]:
logger.error('同步更新与待运行测试计划关联的所有用例树节点信息失败')
return result
else:
logger.info('同步更新与待运行测试计划关联的所有用例树节点信息成功')
for plan_id in self.plan_id_list:
logger.info('正在查询测试计划[ID：%s]相关信息' % plan_id)
result = test_platform_db.select_one_record('SELECT plan_name, browsers, valid_flag FROM `website_ui_test_plan` WHERE id = %s', (plan_id,))
if result[0] and result[1]:
plan_name, browsers, valid_flag = result[1]
browsers = browsers.rstrip(',').split(',')
if valid_flag == '启用':
logger.info('======================开始运行测试计划[名称：%s, ID：%s]======================' % (plan_name, plan_id))
test_plan = TestPlan(plan_id, plan_name, self.project_id, self.project_name, self.home_page, browsers)
result = test_plan.run(debug)
if not result[0]:
if_plan_run_fail = True
plan_id_list_for_failure.append(plan_id)
else:
logger.warn('测试计划已被禁用，跳过执行')
if_plan_run_fail = True
plan_id_list_for_failure.append(plan_id)
continue
elif result[0] and not result[1]:
logger.warn('运行失败:未查询到计划相关信息')
return [False, '运行失败:未查询到计划相关信息']
else:
return [False, '运行失败：%s' % result[1]]
if if_plan_run_fail:
return [False, '测试计划%s运行失败' % str(plan_id_list_for_failure)]
else:
return [True, '']
except Exception as e:
logger.error('%s' % e)
return [False, '运行失败：%s' % e]
def data_received(self, data):
def parse(cmd, size, args):
self.parse_cmd(cmd, size, args, self.transp)
if self.incoming_bytes > 0:
self.data_buffer += data
self.incoming_bytes -= len(data)
logger.debug("waiting for {}".format(self.incoming_bytes))
if self.incoming_bytes <= 0:
if self.incoming_bytes < 0:
logger.warn("incoming_bytes should not be less than zero")
self.incoming_bytes = 0
parse(b'CSTR', 0, self.data_buffer)
self.data_buffer = bytes()
else:
bytes_left = data
while len(bytes_left) > 0:
parsed = bytes_left.split(b' ', 2)
if len(parsed) < 2:
logger.warn('invalid command {}'.format(bytes_left))
return
cmd = parsed[0]
size = parsed[1]
args = parsed[2]
# TODO: handle other commands as well
size_int = int(size.decode())
if cmd == b'CSTR' and size_int > len(args):
self.data_buffer = args
self.incoming_bytes = size_int - len(self.data_buffer)
break
else:
args_temp = args[:size_int]
bytes_left = args[size_int:]
args = args_temp
try:
parse(cmd, size, args)
except Exception as e:
logger.error(e)
for l in traceback.format_tb(e.__traceback__):
logger.error(l)
def data_received(self, data):
def parse(cmd, size, args):
self.parse_cmd(cmd, size, args, self.transp)
if self.incoming_bytes > 0:
self.data_buffer += data
self.incoming_bytes -= len(data)
logger.debug("waiting for {}".format(self.incoming_bytes))
if self.incoming_bytes <= 0:
if self.incoming_bytes < 0:
logger.warn("incoming_bytes should not be less than zero")
self.incoming_bytes = 0
parse(b'CSTR', 0, self.data_buffer)
self.data_buffer = bytes()
else:
bytes_left = data
while len(bytes_left) > 0:
parsed = bytes_left.split(b' ', 2)
if len(parsed) < 2:
logger.warn('invalid command {}'.format(bytes_left))
return
cmd = parsed[0]
size = parsed[1]
args = parsed[2]
# TODO: handle other commands as well
size_int = int(size.decode())
if cmd == b'CSTR' and size_int > len(args):
self.data_buffer = args
self.incoming_bytes = size_int - len(self.data_buffer)
break
else:
args_temp = args[:size_int]
bytes_left = args[size_int:]
args = args_temp
try:
parse(cmd, size, args)
except Exception as e:
logger.error(e)
for l in traceback.format_tb(e.__traceback__):
logger.error(l)
def wrapper(*args, **kw):
try:
return func(*args, **kw)
except Exception as e:
if isinstance(e, ServiceException):
if e.error_code < 500:
logger.warn(e.get_log_msg())
else:
logger.error(e.get_log_msg())
else:
exstr = traceback.format_exc()
logger.error(str(e) + '\n详情：' + exstr)
if throwable:
job_exception = JobServiceException(
e.error_code, e.msg, e.time, e.detail)
raise job_exception
session.rollback()
finally:
session.close()
def run(self, debug):
try:
if_plan_run_fail = False
plan_id_list_for_failure = [] # 存放执行失败、未执行的测试计划的ID及原因简介
logger.info('======================正在同步更新与待运行测试计划关联的所有用例树节点信息======================')
result = self.sync_case_tree_node_info_for_testplans()
if not result[0]:
logger.error('同步更新与待运行测试计划关联的所有用例树节点信息失败')
return result
else:
logger.info('同步更新与待运行测试计划关联的所有用例树节点信息成功')
for plan_id in self.plan_id_list:
logger.info('正在查询测试计划[ID：%s]相关信息' % plan_id)
result = test_platform_db.select_one_record('SELECT plan_name,valid_flag FROM `website_api_test_plan` WHERE id = %s', (plan_id,))
if result[0] and result[1]:
plan_name, valid_flag = result[1]
if valid_flag == '启用':
logger.info('======================开始运行测试计划[名称：%s, ID：%s]======================' % (plan_name, plan_id))
test_plan = TestPlan(plan_id, plan_name, self.project_id, self.project_name, self.protocol, self.host, self.port, self.global_headers)
result = test_plan.run(debug)
if not result[0]:
plan_id_list_for_failure.append('计划ID：%s，失败原因：%s ' % (plan_id, result[1]))
else:
logger.warn('测试计划已被禁用，跳过执行')
plan_id_list_for_failure.append('计划ID：%s，失败原因：%s ' % (plan_id, msg))
continue
elif result[0] and not result[1]:
logger.warn('运行失败:未查询到计划相关信息')
plan_id_list_for_failure.append('计划ID：%s，失败原因：%s ' % (plan_id, msg))
else:
plan_id_list_for_failure.append('计划ID：%s，失败原因：%s ' % (plan_id, result[1]))
if plan_id_list_for_failure:
return [False, '项目运行失败：%s' % ''.join(plan_id_list_for_failure)]
else:
return [True, '项目运行成功']
except Exception as e:
logger.error('%s' % e)
return [False, '运行失败：%s' % e]
def assert_element_text_equal_str(self, selenium_util, element_selector,
target_str):
try:
result = selenium_util.find_element_by_locator_adapter(
element_selector)
if result[0]: #如果找到了
text_of_element = result[1].text
logger.info('获取到的元素[%s]文本内容为：%s' %
(result[1], text_of_element))
if target_str == text_of_element:
return [True, '']
else:
return [
False,
'元素[%s]文本内容不等于给定字符串：%s' % (result[1], text_of_element)
]
else:
logger.warn('未找到元素：%s' % result[1])
return [False, '未找到元素：%s' % result[1]]
except Exception as e:
return [False, '%s' % e]
def __read_file(args_list):
'''读取文件'''
try:
if len(args_list) == 3:
filepath, mode, encoding = args_list
else:
filepath, mode = args_list
encoding = None
co_filepath = sys._getframe().f_code.co_filename
head, tail = os.path.split(co_filepath)
head = os.path.join(head, '../../testdata/')
filepath = os.path.join(head, filepath)
filepath = os.path.normpath(filepath)
msg = '待读取的文件路径为：%s' % filepath
logger.info(msg)
if not os.path.exists(filepath):
logger.warn('文件 %s 不存在' % filepath)
return None
else:
if mode in ('r', 'r+', 'rw'):
file_content = ''
elif mode in ('rb', 'rb+'):
file_content = b''
else:
logger.warn('文件打开方式只支持 r, r+, rw, rb, rb+')
return None
with open(filepath, mode, encoding=encoding) as f:
if file_content == b'':
file_content = f.read()
else:
for line in f:
file_content += line
return file_content
except Exception as e:
logger.error('读取文件 %s 出错：%s' % (filepath, e))
return None
def __run(username, password):
"""
运行函数
:param username: 登录用户名
:param password: 登录密码
:return:
"""
try:
token, leetcode_session = login(username, password)
except Exception as e:
logger.warn('Username=%s登录异常，详情' % username, str(e))
return
if token is None:
logger.warn('Username=%s登录失败' % username)
return
logger.info(
'========================开始爬取Leetcode题目信息============================='
)
process(token, leetcode_session)
logger.info(
'========================Leetcode题目信息爬取完毕============================='
)
def update_mysql(self, case_id, content):
# 现在只支持通过id写入字典格式的数据
# 如果后期需要，可以通过id和给出的列表，更新固定的某些列
logger.info("Write content: {} to case_id: {}".format(
content, case_id))
case_id_content = self.get_data_by_id(case_id)
entry_len = len(case_id_content)
if entry_len == 0:
logger.error("Entry: {} not found.".format(case_id))
sys.exit(1)
# 解析输入的content（是一个字典），把每一个键值对根据数据的结构，更新到数据库中。
# 比如：{"KEY1":"VALUE1"} ,则更新数据库case_id行中，列名为KEY1的列的值为VALUE1
if not isinstance(content, dict):
logger.error("Content: {} is not dict.".format(content))
return
# 写入数据
# 获取mysql链接
elif 0 == len(content):
logger.warn("The length of content: {} is 0. Need not update.")
return
conn = self.connect()
cursor = conn.cursor()
# 循环更新传入数据到数据库
key_value = [
"{} = '{}'".format(key, value) for key, value in content.items()
]
update_str = ', '.join(key_value)
sql = "update {} set {} where id = {};".format(self.table, update_str,
case_id)
logger.info("Execute SQL command: {}".format(sql))
cursor.execute(sql)
conn.commit()
logger.info("Execute SQL command successful!")
# 关闭连接
cursor.close()
conn.close()
def loop(protocol_factory, ip, port):
loop = asyncio.get_event_loop()
# Each client connection will create a new protocol instance
coro = loop.create_server(protocol_factory, ip, port)
try:
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
logger.info('Serving on {}'.format(server.sockets[0].getsockname()))
loop.run_forever()
except Exception:
logger.debug("something happened")
try:
# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
except KeyboardInterrupt:
logger.warn('keyboard interrupt')
exit(1)
def swipe_in_control(self, value, distance, direction='up'):
"""
控件内滑动.待选择数据较多的情况下可使用
:param value: 元素loc
:param distance: 滑动距离 ， 1表示滑动全屏， 2表示滑动一半， 3表示滑动1/3，以此类推
:param direction: 滑动方向
:return:
"""
start_x, start_y = self.get_element_location(value)
height_y, width_x = self.get_element_size(value)
# 屏幕向上滑动
if direction == 'up' or direction == '1':
point_start_x = start_x + width_x / int(distance)
point_start_y = start_y + height_y / int(distance)
self.driver.swipe(
point_start_x,
point_start_y,
point_start_x,
start_y)
# 屏幕向下滑动 ，可用于刷新页面/数据
elif direction == 'down' or direction == '2':
point_start_x = start_x + width_x / int(distance)
point_start_y = start_y + height_y / int(distance)
self.driver.swipe(
point_start_x,
start_y,
point_start_x,
point_start_y)
# 往左滑动
elif direction == 'left' or direction == '3':
pass
# 往右滑动
elif direction == 'right' or direction == '4':
pass
else:
logger.warn('移动方向错误')
def loop(protocol_factory, ip, port):
loop = asyncio.get_event_loop()
# Each client connection will create a new protocol instance
coro = loop.create_server(protocol_factory, ip, port)
try:
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
logger.info('Serving on {}'.format(server.sockets[0].getsockname()))
loop.run_forever()
except Exception:
logger.debug("something happened")
try:
# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
except KeyboardInterrupt:
logger.warn('keyboard interrupt')
exit(1)
def sync_case_tree_node_info_for_testplans(self):
temp_var = ''
def find_case_fullpath(node_id):
nonlocal temp_var
query = "SELECT parent_id, text FROM `website_api_case_tree` WHERE id = %s"
data = (node_id, )
result = test_platform_db.select_one_record(query, data)
if result[0] and result[1]:
parent_id, text = result[1]
temp_var = find_case_fullpath(parent_id) + '->' + text
return temp_var
elif result[0] and not result[1]:
return temp_var
else:
logger.error('查询出错，退出程序')
exit()
try:
logger.info('待运行计划ID列表：%s' % self.plan_id_list)
logger.info('正在查询与测试计划关联的用例树节点')
if len(self.plan_id_list) == 1:
plan_id_list = '(' + str(self.plan_id_list[0]) + ')'
else:
plan_id_list = str(tuple(self.plan_id_list))
query = 'SELECT node_id FROM `website_api_case_tree_test_plan` WHERE plan_id IN ' + plan_id_list + ' GROUP BY node_id'
data = ''
result = test_platform_db.select_many_record(query, data)
if result[0] and result[1]:
records = result[1]
result = [False, '没有找到测试计划关联的节点']
for record in records:
logger.info('正在查找用例树节点信息')
# mysql 查询 count(NULL值) = 0
query = "SELECT t1.text, t1.parent_id, COUNT(t2.id) FROM `website_api_case_tree` AS t1 " \
"LEFT JOIN `website_api_case_tree` AS t2 ON t2.parent_id = t1.id WHERE t1.id = %s"
node_id = int(record[0])
data = (node_id,)
result = test_platform_db.select_one_record(query, data)
if result[0] and result[1]:
text, parent_id, sub_node_num = result[1]
sub_node_num = sub_node_num
node_path = (find_case_fullpath(parent_id) + '').lstrip('->')
logger.info('正在更新测试计划用例树节点关联表记录')
query = "UPDATE website_api_case_tree_test_plan SET node_path='%s', sub_node_num=%s WHERE plan_id IN %s AND node_id = %s"
data = (node_path, sub_node_num, plan_id_list, node_id)
result = test_platform_db.execute_update(query, data)
if not result[0]:
logger.error('更新测试计划用例树节点关联表记录失败：%s' % result[1])
return [False, result[1]]
else:
result = [True, '']
elif result[0] and not result[1]:
logger.warn('用例树节点 %s 不存在' % node_id)
continue
else:
logger.error('查询用例树节点[ID:%s]信息出错，退出程序' % node_id)
exit()
temp_var = ''
return result
elif result[0] and not result[1]:
logger.error('未查找到与测试计划关联的用例树节点')
result = [False, '未查找到与测试计划关联的用例树节点']
else:
logger.error('查找与测试计划关联的用例树节点失败:%s' % result[1])
result = [False, '查找与测试计划关联的用例树节点失败:%s' % result[1]]
return result
except Exception as e:
logger.error('%s' % e)
result = [False, '同步更新与待运行测试计划关联的用例树节点信息失败：%s' % e]
return result
def run_one_node(self, node):
start_time = time.time()
invoices_details = lnclient.listinvoices(
index_offset=node.global_checkpoint,
rpcserver=node.rpcserver,
mock=settings.MOCK_LN_CLIENT)
if node not in self.all_invoices_from_db:
invoice_list_from_db = {}
logger.info("DB has no invoices for this node")
else:
invoice_list_from_db = self.all_invoices_from_db[node]
# example of invoices_details: {"invoices": [], 'first_index_offset': '5', 'last_index_offset': '72'}
invoice_list_from_node = invoices_details['invoices']
self.invoice_count_from_nodes[node] = len(invoice_list_from_node)
if settings.MOCK_LN_CLIENT:
# Here the mock pulls invoices from DB Invoice model, while in prod invoices are pulled from the Lightning node
# 1. Mocked lnclient.listinvoices returns an empty list
# 2. The web front end adds the InvoiceRequest to the DB before it creates the actual invoices with lnclient.addinvoice
# 3. Mocked API lnclient.addinvoice simply fakes converting InvoiceRequest to Invoice and saves to DB
# 4. Here the mocked proces_tasks pulls invoices from DB Invoice model and pretends they came from lnclient.listinvoices
# 5. After X seconds passed based on Invoice created time, here Mock update the Invoice checkpoint to "done" faking a payment
invoice_list_from_node = []
for invoice_obj in Invoice.objects.filter(
lightning_node=node, checkpoint_value="no_checkpoint"):
invoice_request = InvoiceRequest.objects.get(
id=invoice_obj.invoice_request.id)
if invoice_request.lightning_node.id != node.id:
continue
mock_setteled = (invoice_obj.created + timedelta(seconds=3) <
timezone.now())
creation_unixtime = int(
time.mktime(invoice_obj.created.timetuple()))
invoice_list_from_node.append({
"settled":
mock_setteled,
"settle_date":
str(int(time.time())) if mock_setteled else 0,
"state":
"SETTLED" if mock_setteled else "OPEN",
"memo":
invoice_request.memo,
"add_index":
invoice_obj.add_index,
"payment_request":
invoice_obj.pay_req,
"pay_req":
invoice_obj.pay_req, # Old format
"r_hash":
invoice_obj.r_hash,
"creation_date":
str(creation_unixtime),
"expiry":
str(creation_unixtime + 120)
})
retry_mini_map = {
int(invoice['add_index']): False
for invoice in invoice_list_from_node
}
one_hour_ago = timezone.now() - timedelta(hours=1)
recent_invoices = [
i.id for i in invoice_list_from_db.values()
if i.modified > one_hour_ago
]
if len(recent_invoices) == 0:
logger.info("invoice_list_from_db is empty")
else:
logger.info(
"Recent invoice_list_from_db was: {}".format(recent_invoices))
for raw_invoice in invoice_list_from_node:
# Example of raw_invoice:
# {
# 'htlcs': [],
# 'settled': False,
# 'add_index': '5',
# 'value': '1',
# 'memo': '',
# 'cltv_expiry': '40', 'description_hash': None, 'route_hints': [],
# 'r_hash': '+fw...=', 'settle_date': '0', 'private': False, 'expiry': '3600',
# 'creation_date': '1574459849',
# 'amt_paid': '0', 'features': {}, 'state': 'OPEN', 'amt_paid_sat': '0',
# 'value_msat': '1000', 'settle_index': '0',
# 'amt_paid_msat': '0', 'r_preimage': 'd...=', 'fallback_addr': '',
# 'payment_request': 'lnbc...'
# }
created = general_util.unixtime_to_datetime(
int(raw_invoice["creation_date"]))
if created < general_util.now() - settings.INVOICE_RETENTION:
logger.info(
"Got old invoice from listinvoices, skipping... {} is older then retention {}"
.format(created, settings.INVOICE_RETENTION))
continue
add_index_from_node = int(raw_invoice["add_index"])
invoice = invoice_list_from_db.get(add_index_from_node)
if invoice is None:
logger.error(
"Unknown add_index {}".format(add_index_from_node))
logger.error(
"Raw invoice from node was: {}".format(raw_invoice))
if raw_invoice['state'] == "CANCELED":
logger.error("Skipping because invoice is cancelled...")
retry_mini_map[
add_index_from_node] = False # advance global checkpoint
else:
retry_mini_map[
add_index_from_node] = True # try again later
continue
# Validate
if invoice.invoice_request.memo != raw_invoice["memo"]:
logger.error(
"Memo in DB does not match the one in invoice request: db=({}) invoice_request=({})"
.format(invoice.invoice_request.memo, raw_invoice["memo"]))
retry_mini_map[add_index_from_node] = True # try again later
continue
if invoice.pay_req != raw_invoice["payment_request"]:
logger.error(
"Payment request does not match the one in invoice request: db=({}) invoice_request=({})"
.format(invoice.pay_req, raw_invoice["payment_request"]))
retry_mini_map[add_index_from_node] = True # try again later
continue
checkpoint_helper = CheckpointHelper(
node=node,
invoice=invoice,
creation_date=raw_invoice["creation_date"])
if checkpoint_helper.is_checkpointed():
continue
if raw_invoice['state'] == 'CANCELED':
checkpoint_helper.set_checkpoint("canceled")
continue
if raw_invoice['settled'] and (raw_invoice['state'] != 'SETTLED'
or int(raw_invoice['settle_date'])
== 0):
checkpoint_helper.set_checkpoint("inconsistent")
continue
if time.time() > int(raw_invoice['creation_date']) + int(
raw_invoice['expiry']):
checkpoint_helper.set_checkpoint("expired")
continue
if not raw_invoice['settled']:
logger.info("Skipping invoice at {}: Not yet settled".format(
checkpoint_helper))
retry_mini_map[
checkpoint_helper.add_index] = True # try again later
continue
#
# Invoice is settled
#
logger.info(
"Processing invoice at {}: SETTLED".format(checkpoint_helper))
memo = raw_invoice["memo"]
try:
action_details = json_util.deserialize_memo(memo)
except json_util.JsonUtilException:
checkpoint_helper.set_checkpoint("deserialize_failure")
continue
try:
validators.validate_memo(action_details)
except ValidationError as e:
logger.exception(e)
checkpoint_helper.set_checkpoint("memo_invalid")
continue
action = action_details.get("action")
if action:
if action in ["Upvote", "Accept"]:
vote_type = Vote.VOTE_TYPE_MAP[action]
change = settings.PAYMENT_AMOUNT
post_id = action_details["post_id"]
try:
post = Post.objects.get(pk=post_id)
except (ObjectDoesNotExist, ValueError):
logger.error(
"Skipping vote. The post for vote does not exist: {}"
.format(action_details))
checkpoint_helper.set_checkpoint("invalid_post")
continue
user = get_anon_user()
logger.info(
"Creating a new vote: author={}, post={}, type={}".
format(user, post, vote_type))
vote = Vote.objects.create(author=user,
post=post,
type=vote_type)
# Update user reputation
# TODO: reactor score logic to be shared with "mark_fake_test_data.py"
User.objects.filter(pk=post.author.id).update(
score=F('score') + change)
# The thread score represents all votes in a thread
Post.objects.filter(pk=post.root_id).update(
thread_score=F('thread_score') + change)
if vote_type == Vote.ACCEPT:
if "sig" not in action_details:
checkpoint_helper.set_checkpoint("sig_missing")
continue
sig = action_details.pop("sig")
sig = validators.pre_validate_signature(sig)
verifymessage_detail = lnclient.verifymessage(
msg=json.dumps(action_details, sort_keys=True),
sig=sig,
rpcserver=node.rpcserver,
mock=settings.MOCK_LN_CLIENT)
if not verifymessage_detail["valid"]:
checkpoint_helper.set_checkpoint(
"invalid_signiture")
continue
if verifymessage_detail[
"pubkey"] != post.parent.author.pubkey:
checkpoint_helper.set_checkpoint(
"signiture_unauthorized")
continue
if change > 0:
# First, un-accept all answers
for answer in Post.objects.filter(
parent=post.parent, type=Post.ANSWER):
if answer.has_accepted:
Post.objects.filter(pk=answer.id).update(
vote_count=F('vote_count') - change,
has_accepted=False)
# There does not seem to be a negation operator for F objects.
Post.objects.filter(pk=post.id).update(
vote_count=F('vote_count') + change,
has_accepted=True)
Post.objects.filter(pk=post.root_id).update(
has_accepted=True)
else:
# TODO: change "change". here change is set to payment ammount, so does not make sense to be called change
# TODO: detect un-accept attempt and raise "Un-accept not yet supported"
raise Exeption(
"Payment ammount has to be positive")
else:
Post.objects.filter(pk=post.id).update(
vote_count=F('vote_count') + change)
# Upvote on an Aswer is the trigger for potentian bounty awards
if post.type == Post.ANSWER and post.author != get_anon_user(
):
award_bounty(question_post=post.parent)
checkpoint_helper.set_checkpoint("done",
action_type="upvote",
action_id=post.id)
elif action == "Bounty":
valid = True
for keyword in ["post_id", "amt"]:
if keyword not in action_details:
logger.warn(
"Bounty invalid because {} is missing".format(
keyword))
valid = False
if not valid:
logger.warn("Could not start Bounty: bounty_invalid")
checkpoint_helper.set_checkpoint("bounty_invalid")
continue
post_id = action_details["post_id"]
amt = action_details["amt"]
try:
post_obj = Post.objects.get(pk=post_id)
except (ObjectDoesNotExist, ValueError):
logger.error(
"Bounty invalid because post {} does not exist".
format(post_id))
checkpoint_helper.set_checkpoint(
"bounty_invalid_post_does_not_exist")
continue
logger.info("Starting bounty for post {}!".format(post_id))
new_b = Bounty(
post_id=post_obj,
amt=amt,
activation_time=timezone.now(),
)
new_b.save()
checkpoint_helper.set_checkpoint("done",
action_type="bonty",
action_id=post_id)
else:
logger.error("Invalid action: {}".format(action_details))
checkpoint_helper.set_checkpoint("invalid_action")
continue
else:
# Posts do not include the "action" key to save on memo space
logger.info("Action details {}".format(action_details))
if "sig" in action_details:
sig = action_details.pop("sig")
sig = validators.pre_validate_signature(sig)
verifymessage_detail = lnclient.verifymessage(
msg=json.dumps(action_details, sort_keys=True),
sig=sig,
rpcserver=node.rpcserver,
mock=settings.MOCK_LN_CLIENT)
if not verifymessage_detail["valid"]:
checkpoint_helper.set_checkpoint("invalid_signiture")
continue
pubkey = verifymessage_detail["pubkey"]
else:
pubkey = "Unknown"
if "parent_post_id" in action_details:
# Find the parent.
try:
parent_post_id = int(action_details["parent_post_id"])
parent = Post.objects.get(pk=parent_post_id)
except (ObjectDoesNotExist, ValueError):
logger.error(
"The post parent does not exist: {}".format(
action_details))
checkpoint_helper.set_checkpoint("invalid_parent_post")
continue
title = parent.title
tag_val = parent.tag_val
else:
title = action_details["title"]
tag_val = action_details["tag_val"]
parent = None
user, created = User.objects.get_or_create(pubkey=pubkey)
post = Post(
author=user,
parent=parent,
type=action_details["post_type"],
title=title,
content=action_details["content"],
tag_val=tag_val,
)
# TODO: Catch failures when post title is duplicate (e.g. another node already saved post)
post.save()
# New Answer is the trigger for potentian bounty awards
if post.type == Post.ANSWER and user != get_anon_user():
award_bounty(question_post=post.parent)
# Save tags
if "tag_val" in action_details:
tags = action_details["tag_val"].split(",")
for tag in tags:
tag_obj, created = Tag.objects.get_or_create(name=tag)
if created:
logger.info("Created a new tag: {}".format(tag))
tag_obj.count += 1
post.tag_set.add(tag_obj)
tag_obj.save()
post.save()
checkpoint_helper.set_checkpoint("done",
action_type="post",
action_id=post.id)
# advance global checkpoint
new_global_checkpoint = None
for add_index in sorted(retry_mini_map.keys()):
retry = retry_mini_map[add_index]
if retry:
break
else:
logger.info("add_index={} advances global checkpoint".format(
add_index))
new_global_checkpoint = add_index
if new_global_checkpoint:
node.global_checkpoint = new_global_checkpoint
node.save()
logger.info(
"Saved new global checkpoint {}".format(new_global_checkpoint))
processing_wall_time = time.time() - start_time
logger.info("Processing node {} took {:.3f} seconds".format(
node.node_name, processing_wall_time))
return processing_wall_time
% running_plan_name)
result = running_plan.run(is_debug)
if not is_debug:
if result[0]:
logger.info('执行成功，正在更新数据库运行计划的运行状态')
else:
logger.info('执行失败，正在更新数据库运行计划的运行状态')
update_query = "UPDATE `website_running_plan` SET running_status ='%s', remark='%s' WHERE running_plan_num= %s"
data = (result[1], result[2].replace("'", '\"'),
running_plan_num)
result = test_platform_db.execute_update(
update_query, data)
if not result[0]:
logger.error('更新数据库运行计划的运行状态失败：%s' % result[1])
else:
logger.warn('执行失败，运行计划已被禁用')
elif result[0] and not result[1]:
logger.error('未查询到运行计划相关的信息')
else:
logger.error('查询运行计划相关信息失败：%s' % result[1])
except Exception as e:
logger.error('%s' % e)
elif run_mode.lower() == 'rap':
logger.info('运行所有项目')
elif run_mode.lower() == 'debug':
logger.info('调试模式')
try:
input_id = ''
while not input_id.isdigit():
input_id = input('请输入用例ID、套件ID：')
if not input_id.isdigit():
def FDEL(client, args):
logger.warn('unimplemented FDEL')
pass
import logging
import time
from common.log import logger
import common.network
import argparse
logfile = '/tmp/storeit-server.log'
parser = argparse.ArgumentParser(description='StoreIt backend server.')
parser.add_argument('-l', '--log', nargs='?', const=logfile,
help='log to a file (default is {})'.format(logfile))
args = parser.parse_args()
if args.log is not None:
logging.basicConfig(filename=args.log, level=logging.DEBUG)
try:
database.__init__()
database.find_user('nonexistent', 'user')
except Exception:
logger.warn('Looks like postgre daemon is not running. It will be started')
subprocess.call("./database/postgre.sh")
time.sleep(0.2)
database.__init__()
try:
common.network.loop(network.NetManager, '0', 7641)
except Exception:
logger.error("exception")
