def main(conf):
# 启动调度器
scheduler = Scheduler(conf)
scheduler.start()
# 启动输入流程
scheduler_server = SchedulerServer(conf, scheduler)
scheduler_server.start()
def signal_handler(signalnum, frame):
conf['log'].info("received a signal:%s" % signalnum)
scheduler_server.stop("stops")
exit(1)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGUSR1, lambda a, b: profiling_signal_handler("scheduler", a, b))
try:
handler = ScheduleHandler(scheduler)
processor = SchedulerService.Processor(handler)
transport = TSocket.TServerSocket(port=conf['server']['port'])
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory, daemon=True)
server.setNumThreads(conf['server']['server_thread_num'])
server.serve()
except Exception as e:
log.error("调度器启动失败...")
log.exception(e)
conf['log'].error(str(traceback.format_exc()))
scheduler_server.stop("fail")
scheduler_server.stop("success")
def wait_eleVisible(self,
locator,
wait_times=30,
poll_frequency=0.5,
doc=""):
"""'
等待元素可见
:param locator: 元素定位，元祖形式
:param wait_times: 最长等待时间
:param poll_frequency: 轮询间隔
:param doc: 模块名_页面名称_操作名称
:return:
"""
try:
# 开始等待时间
start = time.time()
WebDriverWait(self.driver, wait_times, poll_frequency).until(
EC.visibility_of_element_located((locator[1])))
# 结束等待的时间点
end = time.time()
# 求差值
wait_time = round(end - start, 3)
log.info("等待" + '"' + locator[0] + '"加载成功，用时{}秒'.format(wait_time))
except Exception as e:
log.error("页面加载超时！判断依据:" + '"' + locator[0] + '"')
# 截图
self.save_screenshot(doc)
raise e
def check_interface(self, content):
try:
if isinstance(content, dict):
if 'interface_name' in content:
log.info("正在测试接口：%s" % content['interface_name'])
log.info("请求的参数为：%s" % str(content))
data1 = {"content": json.dumps(content)}
res = self.send_post(url1, data1)
token = res[1]['token']
data2 = {"content": json.dumps(content), "token": token}
res2 = self.send_post(url2, data2)
return res2
else:
content_dict = eval(content)
if 'interface_name' in content_dict:
log.info("正在测试接口：%s" % content_dict['interface_name'])
log.info("请求的参数为：%s" % content)
data1 = {"content": json.dumps(content_dict)}
res = self.send_post(url1, data1)
token = res[1]['token']
data2 = {
"content": json.dumps(content_dict),
"token": token
}
res2 = self.send_post(url2, data2)
return res2
except Exception as msg:
log.error("错误信息是：%s" % msg)
raise
def run(self):
log.debug("starting input thread")
job_num = 0
while self.running and self.input_tube:
try:
job = self.beanstalk.reserve(self.input_tube, 3)
if job:
job_num += 1
body = job.body
resp = None
job.delete()
if self.processor is not None:
topic_id = None
try:
if type(self.processor).__name__ in (
'ExtractorProccessor',
'SingleSrcMergerProccessor'):
resp, topic_id = self.processor.do_task(body)
else:
resp = self.processor.do_task(body)
except Exception, e:
log.error("Process failed. " +
traceback.format_exc())
if resp is not None:
self.output_msg(resp, topic_id)
else:
self.log.debug(current_process().name +
" : no msg from : %s" % (self.input_tube))
def upload_file(self, file_path, doc):
"""
谷歌上传文件
:param file_path: 文件地址
:param doc: 模块名_页面名称_操作名称
:return:
"""
time.sleep(2)
try:
# 一级窗口（打开）
open_win = win32gui.FindWindow("#32770", "打开")
# 二级窗口
combo_box_ex32 = win32gui.FindWindowEx(open_win, 0, "ComboBoxEx32",
None)
# 三级窗口
combo_box = win32gui.FindWindowEx(combo_box_ex32, 0, "ComboBox",
None)
# 四级窗口（文件名）
edit = win32gui.FindWindowEx(combo_box, 0, "Edit", None)
# 二级窗口（打开）
button = win32gui.FindWindowEx(open_win, 0, "Button", "打开(&O)")
# 操作
win32gui.SendMessage(edit, win32con.WM_SETTEXT, None, file_path)
time.sleep(1)
win32gui.SendMessage(open_win, win32con.WM_COMMAND, 1, button)
log.info("上传本地文书：{}".format(file_path))
except Exception as e:
log.error("上传文书失败！路径:{}".format(file_path))
self.save_screenshot(doc)
raise e
def process_mq_cleaning(conf):
global running
signal.signal(signal.SIGTERM, exit_isr)
signal.signal(signal.SIGINT, exit_isr)
# This is only used by thrift, each process has its own EntityExtractor instance
singletons[EntityExtractor.__name__] = EntityExtractor(conf)
singletons[EEWorkerPool.__name__] = EEWorkerPool(
init_count=conf['server']['server_process_num'], conf=conf)
worker_pool = singletons[EEWorkerPool.__name__]
worker_pool.start_all()
heartbeat_thread = HeartbeatThread("entity_extractor", conf)
heartbeat_thread.start()
try:
handler = EntityExtractorHandler(conf, worker_pool)
processor = EntityExtractorService.Processor(handler)
transport = TSocket.TServerSocket(port=conf['server']['port'])
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = InterruptableThreadPoolServer(processor, transport, tfactory,
pfactory)
server.setNumThreads(1)
server.serve()
except Exception, e:
log.error(str(e))
log.error(traceback.format_exc())
def add_topic(self, topic_info):
''''''
resp = toolsutil.result()
try:
id = topic_info['id']
table_name = topic_info['table_name']
schema = topic_info['schema']
primary_keys = topic_info['primary_keys']
except:
log.error("topic_info error,\ttopic_info:%s\tret:%s" % (topic_info, traceback.format_exc()))
resp['MSG'] = "topic_info error,\ttopic_info:%s\tret:%s" % (topic_info, traceback.format_exc())
resp['CODE'] = -10000
return resp
sql = '''INSERT INTO topic (id, name, table_name, `schema`, primary_keys) VALUES(%s,%s,%s,%s,%s)'''
param = (id, topic_info.get('name', ''), table_name, schema, primary_keys)
mysql_db = get_mysql_conn(self.conf)
try:
cursor = mysql_db.cursor()
cursor.execute(sql, param)
except:
resp['MSG'] = "insert data into mysql error,\ttopic_info:%s\tsql:%s\tret:%s" % (
topic_info, sql, traceback.format_exc())
resp['CODE'] = -10000
mysql_db.rollback()
mysql_db.close()
return resp
mysql_db.commit()
mysql_db.close()
resp['MSG'] = "insert data into mysql success, topic_info:%s, sql:%s" % (topic_info, sql)
return resp
def select(self,
site,
url_format,
limit,
start=0,
extra_filter='{}',
tube_name='download_rsp'):
ret = RetStatus()
first_result = self.select_handler.select_webpage(
site, url_format, 1, start, extra_filter)
if not first_result:
ret.status = 0
return ret
else:
try:
self.beanstalk.put(tube_name, self.to_string(first_result[0]))
ret.status = 1
if limit > 1:
# 创建守护进程,完成发送剩下数据
back_t = threading.Thread(target=self.back_work,
args=(site, url_format, limit,
start, extra_filter,
tube_name))
back_t.setDaemon(True)
back_t.start()
return ret
except Exception, e:
log.error("put msg from:{0}\tresult:{1}".format(
tube_name, str(e)))
ret.status = 0
ret.errormessage = e.message
return ret
def load_extractors(self, topic_id=-1):
'''加载解析器'''
mysql_db = get_mysql_conn(self.conf)
if topic_id != -1:
sql = '''select f_topic_id, f_module_path, f_module_name, f_extractor_name from extractors where f_topic_id=%s'''
else:
sql = '''select f_topic_id, f_module_path, f_module_name, f_extractor_name from extractors'''
try:
cursor = mysql_db.cursor()
if topic_id != -1:
param = [str(topic_id)]
cursor.execute(sql, param)
else:
cursor.execute(sql)
result = cursor.fetchall()
mysql_db.close()
for item in result:
extractor_info = dbrecord_to_dict(cursor.description, item)
topic_id = int(extractor_info.get("f_topic_id", -1))
self.modules_config[topic_id] = extractor_info
except:
log.error("query data from mysql error, sql:%s, ret:%s" % (sql, traceback.format_exc()))
mysql_db.close()
return None
log.info("load_modules_success, moudle:%s" % json.dumps(self.modules_config))
return self.modules_config.keys()
def open_browser(url=None):
set_headless = config.conf.get_str("env", "head_less")
set_browser = config.conf.get_str("env", "browser")
project_url = config.conf.get_str("env", "url")
url = project_url if url is None else url
try:
if set_browser == "Chrome":
options = webdriver.ChromeOptions()
prefs = {
"download.prompt_for_download": False,
'download.default_directory': config.downloads_dir, # 下载目录
'profile.default_content_settings.popups':
0, # 设置为0，禁止弹出窗口
'safebrowsing.enabled': True,
}
options.add_experimental_option('prefs', prefs)
if set_headless == "True":
options.add_argument('--headless')
options.add_argument('--disable-gpu')
driver = webdriver.Chrome(options=options,
executable_path=config.driver_path)
driver.maximize_window()
driver.get(url=url)
return driver
else:
raise Exception("暂不支持谷歌以外浏览器")
except Exception as e:
log.error("拉起浏览器失败")
raise e
def pytest_runtest_makereport(item, call):
# execute all other hooks to obtain the report object
outcome = yield
rep = outcome.get_result()
# we only look at actual failing test calls, not setup/teardown
if rep.when == "call" and not rep.passed:
if 'global_test_env' in item.fixturenames:
# download log in here
try:
log_name = item.funcargs["global_test_env"].backup_all_logs(item.name)
job = item.funcargs["request"].config.getoption("--job")
if job is None:
log_url = os.path.join(item.funcargs["global_test_env"].cfg.bug_log, log_name)
else:
log_url = "http://{}:8080/job/PlatON/job/run/{}/artifact/logs/{}".format(socket.gethostbyname(socket.gethostname()), job, log_name)
allure.attach('{}'.format(log_url), 'env log', allure.attachment_type.URI_LIST)
except Exception as e:
log.info("exception:{}".format(e))
# Record block number
try:
if item.funcargs["global_test_env"].running:
env_status = "node blocks:{}".format(item.funcargs["global_test_env"].block_numbers())
else:
env_status = "node runnings:{}".format(["{}:{}".format(node.node_mark, node.running) for node in
item.funcargs["global_test_env"].get_all_nodes()])
log.info(env_status)
allure.attach(env_status, "env status", allure.attachment_type.TEXT)
except Exception as e:
log.info("get block exception:{}".format(e))
else:
log.error("This case does not use global_test_env")
def count_downloadsFiles():
try:
count = len(os.listdir(config.downloads_dir))
log.info("获取下载目录文件数：{}".format(count))
return count
except Exception as e:
log.error("获取下载目录文件失败")
raise e
def join(self):
resp = self.to_main_queue.get(block=True)
assert type(resp) == ResponsePacket
if resp.is_succ():
self.process.join()
log.debug("main proc : %s joined" % str(self.process.name))
else:
log.error("Failed to stop subprocess " + str(self.process.name))
def do_js(self, locator, doc=""):
ele = self.get_element(locator, doc=doc)
try:
self.driver.execute_script(
'arguments[0].removeAttribute("readonly")', ele)
log.info('去除"' + locator[0] + '"只读属性')
except Exception as e:
log.error('去除"' + locator[0] + '"只读属性失败！')
raise e
def __init():
cmd = '{} --datadir {} init {}'.format(self.remote_bin_file, self.remote_data_dir, self.remote_genesis_file)
result = self.run_ssh(cmd)
# todo ：fix init complete
# Adding a query here can only alleviate the problem of starting deployment without initialization.
self.run_ssh("ls {}".format(self.remote_data_dir))
if len(result) > 0:
log.error(failed_msg.format(self.node_mark, "init", result[0]))
raise Exception("Init failed:{}".format(result[0]))
log.debug("node-{} init success".format(self.node_mark))
def open_software(self, software_path, doc=""):
try:
# 启动软件
self.app = Application('uia').start(software_path)
sleep(2)
app2 = Application('uia').connect(path=software_path)
dlg = app2['MicroSIP']
log.info(doc + "开启成功")
except Exception as e:
log.error(doc + "开启失败")
raise e
def insert_extractor(self, extractor_info):
'''往数据库中写入解析器'''
resp = toolsutil.result()
try:
topic_id = extractor_info['topic_id']
target_dir_name = extractor_info['target_dir_name']
extractor_name = extractor_info.get('extractor_name', '')
except:
log.error("extractor_info error,\textractor_info:%s\tret:%s" % (extractor_info, traceback.format_exc()))
resp['MSG'] = "extractor_info error,\textractor_info:%s\tret:%s" % (extractor_info, traceback.format_exc())
resp['CODE'] = -10000
return resp
moudule_path = "extractors/" + target_dir_name + "/" + target_dir_name + "_extractor.py"
if target_dir_name:
moudule_name = target_dir_name[0].upper() + target_dir_name[1:] + "Extractor"
else:
moudule_name = "TemplateExtractor"
# 1 创建解析器默认文件和默认代码
target_dir = self.basic_path + "i_entity_extractor/extractors/" + target_dir_name
ret = self.copy_files(self.src_dir, target_dir)
if not ret:
resp['MSG'] = "copy_files_error,\textractor_info:%s\tret:%s" % (extractor_info, traceback.format_exc())
resp['CODE'] = -10000
return resp
target_file = target_dir + "/" + target_dir_name + "_extractor.py"
source_file = target_dir + "/template_extractor.py"
open(target_file, "wb").write(open(source_file, "rb").read().replace("TemplateExtractor", moudule_name))
os.remove(source_file)
# 2 在数据库中添加解析器配置
sql = '''INSERT INTO extractors (f_topic_id, f_module_path, f_module_name, f_extractor_name) VALUES(%s,%s,%s,%s)'''
mysql_db = get_mysql_conn(self.conf)
try:
cursor = mysql_db.cursor()
cursor.execute(sql, (topic_id, moudule_path, moudule_name, extractor_name))
except:
resp['MSG'] = "insert data into mysql error,\textractor_info:%s\tsql:%s\tret:%s" % (
extractor_info, sql, traceback.format_exc())
resp['CODE'] = -10000
mysql_db.rollback()
mysql_db.close()
return resp
mysql_db.commit()
mysql_db.close()
resp['MSG'] = "insert data into mysql success, extractor_info:%s, sql:%s" % (extractor_info, sql)
return resp
def test_register_success(self, case):
allure.dynamic()
with allure.step("第一步：准备数据"):
with allure.step("接口地址"):
url = conf.get("env", "host") + case["url"]
allure.attach(
body=url,
name="接口请求地址") # , extension=allure.attachment_type.TEXT
with allure.step("请求方法"):
method = case["method"]
allure.attach(body=method,
name="接口请求方法",
extension=allure.attachment_type.TEXT)
headers = eval(conf.get("env", "headers"))
with allure.step("请求数据"):
case["data"] = replace_data(case["data"])
json = case["data"]
allure.attach(body=json,
name="用例数据",
extension=allure.attachment_type.JSON)
with allure.step("预期结果"):
expected = eval(case["expected"])
allure.attach(body=case["expected"],
name="预期结果",
extension=allure.attachment_type.TEXT)
row = case["case_id"] + 1
# 发送请求
with allure.step("第二步：获取响应结果"):
response = self.request.send_request(url=url,
method=method,
headers=headers,
json=eval(json))
allure.attach(str(response.json()), "响应结果",
allure.attachment_type.JSON)
res = response.json()
with allure.step("第三步：测试结果"):
try:
assert expected["code"] == res["code"]
assert expected["msg"] == res["msg"]
except AssertionError as e:
self.excel1.write_data(row=None, column=8, value="未通过")
log.error(f"用例{case['title']},测试未通过")
log.exception(e)
raise e
else:
self.excel1.write_data(row=row, column=8, value="通过")
log.info(f"用例{case['title']},测试通过")
allure.attach(f"{case['title']}用例测试通过", "测试结果",
allure.attachment_type.TEXT)
def save_screenshot(self, doc=""):
"""
截图
:param doc:
:return:
"""
# 图片名称：模块名_页面名称_操作名称_年-月-日_时分秒.png
file_path = config.screenshot_dir + \
"\\{0}_{1}.png".format(doc, time.strftime("%Y-%m-%d-%H-%M-%S"))
try:
self.driver.save_screenshot(file_path)
log.info("截屏成功，图片路径为{}".format(file_path))
except Exception as e:
log.error("截图失败")
raise e
def start_handler(self, packet, from_main_queue, to_main_queue):
try:
self.on_start()
self.input_thread = InputThread(conf=self.conf,
proc_name=self.proc.process.name,
processor=self.processor)
self.input_thread.start()
self.proc.running = True
to_main_queue.put(ResponsePacket(packet, True))
except Exception as e:
log.error(str(traceback.format_exc()))
to_main_queue.put(ResponsePacket(packet, False))
raise e
log.debug("%s : invoking start handler for process " %
str(self.proc.process.name))
def clear_text(self, locator, doc=""):
"""
清空输入内容
:param locator: 元素定位
:param doc: 模块名_页面名称_操作名称
:return:
"""
try:
ele = self.get_element(locator)
ele.clear()
log.info('清空"' + locator[0] + '"文本')
except Exception as e:
log.error('清空"' + locator[0] + '"文本失败！')
self.save_screenshot(doc)
raise e
def get_element_attribute(self, locator, attr, doc=""):
"""
获取元素属性
:param locator: 元素定位
:param attr: 属性名称
:param doc: 模块名_页面名称_操作名称
:return:
"""
ele = self.get_element(locator, doc=doc)
try:
return ele.get_attribute(attr)
except Exception as e:
log.error("获取元素属性失败！定位:{}".format(locator[0]))
# 截图
self.save_screenshot(doc)
raise e
def click_element(self, locator, doc=""):
"""
点击操作
:param locator: 元素名称
:param doc: 模块名_页面名称_操作名称
:return:
"""
ele = self.get_element(locator, doc=doc)
try:
ele.click()
log.info("点击：{0}".format(locator[0]))
except Exception as e:
log.error("点击" + locator[0] + "失败！")
# 截图
self.save_screenshot(doc)
raise e
def get_elements(self, locator, doc=""):
"""
查找元素（多个）
:param locator: 元素定位
:param doc: 模块名_页面名称_操作名称
:return:
"""
try:
ele = self.driver.find_elements(*locator[1])
log.info("定位" + '"' + locator[0] + '"成功')
return ele
except Exception as e:
log.error("定位" + '"' + locator[0] + '"失败')
# 截图
self.save_screenshot(doc)
raise e
def upload_file(file_path, file, doc='', *args):
try:
# 选择文件
app = pywinauto.Desktop()
dlg = app["打开"]
dlg["Toolbar3"].click()
send_keys(file_path)
send_keys("{VK_RETURN}")
dlg["文件名(&N):Edit"].type_keys('"{}"'.format(file))
for i in args:
send_keys('"{}"'.format(i))
dlg["打开(&O)"].click()
log.info(doc + "上传文件成功")
except Exception as e:
log.error(doc + "上传文件失败")
raise e
def init_connection_pool(conf):
global pool_holder
if pool_holder.pool is not None:
# raise Exception("Trying to initialize mysql connection pool twice")
return # Silently success
try:
pool_holder.pool = PersistentDB.PersistentDB(MySQLdb, 1000,
host=conf['MYSQL']['host'],
port=conf['MYSQL']['port'],
user=conf['MYSQL']['username'],
passwd=conf['MYSQL']['password'],
db=conf['MYSQL']['dbname'],
charset='utf8',
setsession=['set autocommit = 1'])
except Exception, e:
log.error("failed to initialize mysql connection pool, message : %s" % (e.message))
log.error(traceback.format_exc())
def __init__(self, conf, processor=None, proc_name=None):
threading.Thread.__init__(self)
self.running = True
self.proc_name = proc_name # Only for logging
self.input_tube = conf['beanstalk_conf']['input_tube']
self.beanstalk = PyBeanstalk(conf['beanstalk_conf']['host'],
conf['beanstalk_conf']['port'])
self.out_beanstalk = PyBeanstalk(conf['beanstalk_conf']['host'],
conf['beanstalk_conf']['port'])
self.output_tube = conf['beanstalk_conf']['output_tube']
self.topic_output_tubes = {}
self.topic_output_tubes.setdefault('default', [])
"""
output_tube = ["default_out", "only_special_out:1,2,3:exclusive", "special_out:4", ":5:exclusive"]
topic_id:1,2,3只会用到only_special_out
topic_id:4 会进入special_out和default_out
topic_id:5 不会进入队列
topic_id:else 用用default_out队列
"""
if type(self.output_tube) == list:
for tube_def in self.output_tube:
tube_def = tube_def.strip()
if len(tube_def.split(":")) < 2:
self.topic_output_tubes['default'].append(tube_def)
else:
elements = [a.strip() for a in tube_def.split(':')]
tube_name = elements[0]
topic_ids = [
int(a.strip()) for a in elements[1].split(',')
]
exclusive = False
if len(elements) == 3 and elements[2] == 'exclusive':
exclusive = True
for topic_id in topic_ids:
self.topic_output_tubes.setdefault(topic_id, [])
self.topic_output_tubes[topic_id].append(
(tube_name, exclusive))
else:
self.topic_output_tubes['default'].append(self.output_tube)
self.log = log
if processor is None:
log.error("Processor not given !")
raise Exception("Processor not given !")
else:
self.processor = processor
def get_text(self, locator, doc=""):
"""
获取元素的文本内容
:param locator: 元素定位
:param doc: 模块名_页面名称_操作名称
:return:
"""
try:
ele = self.get_element(locator, doc=doc)
text = ele.text
log.info('获取"' + locator[0] + '"文本，内容为:{}'.format(text))
return text
except Exception as e:
log.error('获取"' + locator[0] + '"文本内容失败！')
# 截图
self.save_screenshot(doc)
raise e
def input_text(self, locator, text, doc=""):
"""
输入操作
:param locator: 元素定位
:param text: 输入的内容
:param doc: 模块名_页面名称_操作名称
:return:
"""
ele = self.get_element(locator, doc=doc)
try:
ele.send_keys(text)
log.info('向"' + locator[0] + '"输入内容:{}'.format(text))
except Exception as e:
log.error('向"' + locator[0] + '"输入内容:{}失败！'.format(text))
# 截图
self.save_screenshot(doc)
raise e
def start(self):
self.process.start()
self.from_main_queue.put(StartPacket(), block=True)
resp = self.to_main_queue.get(block=True)
assert type(resp) == ResponsePacket
if resp.is_succ():
log.debug("main proc : %s started successfully" %
str(self.process.name))
self.running = True
return True
else:
log.error("main proc : Failed to start subprocess " +
str(self.process.name))
self.running = False
#保证子进程启动不成功的is_alive状态为false
self.process.join()
return False
