def main(conf): # 启动调度器 scheduler = Scheduler(conf) scheduler.start() # 启动输入流程 scheduler_server = SchedulerServer(conf, scheduler) scheduler_server.start() def signal_handler(signalnum, frame): conf['log'].info("received a signal:%s" % signalnum) scheduler_server.stop("stops") exit(1) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGUSR1, lambda a, b: profiling_signal_handler("scheduler", a, b)) try: handler = ScheduleHandler(scheduler) processor = SchedulerService.Processor(handler) transport = TSocket.TServerSocket(port=conf['server']['port']) tfactory = TTransport.TBufferedTransportFactory() pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory, daemon=True) server.setNumThreads(conf['server']['server_thread_num']) server.serve() except Exception as e: log.error("调度器启动失败...") log.exception(e) conf['log'].error(str(traceback.format_exc())) scheduler_server.stop("fail") scheduler_server.stop("success")
def wait_eleVisible(self, locator, wait_times=30, poll_frequency=0.5, doc=""): """' 等待元素可见 :param locator: 元素定位,元祖形式 :param wait_times: 最长等待时间 :param poll_frequency: 轮询间隔 :param doc: 模块名_页面名称_操作名称 :return: """ try: # 开始等待时间 start = time.time() WebDriverWait(self.driver, wait_times, poll_frequency).until( EC.visibility_of_element_located((locator[1]))) # 结束等待的时间点 end = time.time() # 求差值 wait_time = round(end - start, 3) log.info("等待" + '"' + locator[0] + '"加载成功,用时{}秒'.format(wait_time)) except Exception as e: log.error("页面加载超时!判断依据:" + '"' + locator[0] + '"') # 截图 self.save_screenshot(doc) raise e
def check_interface(self, content): try: if isinstance(content, dict): if 'interface_name' in content: log.info("正在测试接口:%s" % content['interface_name']) log.info("请求的参数为:%s" % str(content)) data1 = {"content": json.dumps(content)} res = self.send_post(url1, data1) token = res[1]['token'] data2 = {"content": json.dumps(content), "token": token} res2 = self.send_post(url2, data2) return res2 else: content_dict = eval(content) if 'interface_name' in content_dict: log.info("正在测试接口:%s" % content_dict['interface_name']) log.info("请求的参数为:%s" % content) data1 = {"content": json.dumps(content_dict)} res = self.send_post(url1, data1) token = res[1]['token'] data2 = { "content": json.dumps(content_dict), "token": token } res2 = self.send_post(url2, data2) return res2 except Exception as msg: log.error("错误信息是:%s" % msg) raise
def run(self): log.debug("starting input thread") job_num = 0 while self.running and self.input_tube: try: job = self.beanstalk.reserve(self.input_tube, 3) if job: job_num += 1 body = job.body resp = None job.delete() if self.processor is not None: topic_id = None try: if type(self.processor).__name__ in ( 'ExtractorProccessor', 'SingleSrcMergerProccessor'): resp, topic_id = self.processor.do_task(body) else: resp = self.processor.do_task(body) except Exception, e: log.error("Process failed. " + traceback.format_exc()) if resp is not None: self.output_msg(resp, topic_id) else: self.log.debug(current_process().name + " : no msg from : %s" % (self.input_tube))
def upload_file(self, file_path, doc): """ 谷歌上传文件 :param file_path: 文件地址 :param doc: 模块名_页面名称_操作名称 :return: """ time.sleep(2) try: # 一级窗口(打开) open_win = win32gui.FindWindow("#32770", "打开") # 二级窗口 combo_box_ex32 = win32gui.FindWindowEx(open_win, 0, "ComboBoxEx32", None) # 三级窗口 combo_box = win32gui.FindWindowEx(combo_box_ex32, 0, "ComboBox", None) # 四级窗口(文件名) edit = win32gui.FindWindowEx(combo_box, 0, "Edit", None) # 二级窗口(打开) button = win32gui.FindWindowEx(open_win, 0, "Button", "打开(&O)") # 操作 win32gui.SendMessage(edit, win32con.WM_SETTEXT, None, file_path) time.sleep(1) win32gui.SendMessage(open_win, win32con.WM_COMMAND, 1, button) log.info("上传本地文书:{}".format(file_path)) except Exception as e: log.error("上传文书失败!路径:{}".format(file_path)) self.save_screenshot(doc) raise e
def process_mq_cleaning(conf): global running signal.signal(signal.SIGTERM, exit_isr) signal.signal(signal.SIGINT, exit_isr) # This is only used by thrift, each process has its own EntityExtractor instance singletons[EntityExtractor.__name__] = EntityExtractor(conf) singletons[EEWorkerPool.__name__] = EEWorkerPool( init_count=conf['server']['server_process_num'], conf=conf) worker_pool = singletons[EEWorkerPool.__name__] worker_pool.start_all() heartbeat_thread = HeartbeatThread("entity_extractor", conf) heartbeat_thread.start() try: handler = EntityExtractorHandler(conf, worker_pool) processor = EntityExtractorService.Processor(handler) transport = TSocket.TServerSocket(port=conf['server']['port']) tfactory = TTransport.TBufferedTransportFactory() pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = InterruptableThreadPoolServer(processor, transport, tfactory, pfactory) server.setNumThreads(1) server.serve() except Exception, e: log.error(str(e)) log.error(traceback.format_exc())
def add_topic(self, topic_info): '''''' resp = toolsutil.result() try: id = topic_info['id'] table_name = topic_info['table_name'] schema = topic_info['schema'] primary_keys = topic_info['primary_keys'] except: log.error("topic_info error,\ttopic_info:%s\tret:%s" % (topic_info, traceback.format_exc())) resp['MSG'] = "topic_info error,\ttopic_info:%s\tret:%s" % (topic_info, traceback.format_exc()) resp['CODE'] = -10000 return resp sql = '''INSERT INTO topic (id, name, table_name, `schema`, primary_keys) VALUES(%s,%s,%s,%s,%s)''' param = (id, topic_info.get('name', ''), table_name, schema, primary_keys) mysql_db = get_mysql_conn(self.conf) try: cursor = mysql_db.cursor() cursor.execute(sql, param) except: resp['MSG'] = "insert data into mysql error,\ttopic_info:%s\tsql:%s\tret:%s" % ( topic_info, sql, traceback.format_exc()) resp['CODE'] = -10000 mysql_db.rollback() mysql_db.close() return resp mysql_db.commit() mysql_db.close() resp['MSG'] = "insert data into mysql success, topic_info:%s, sql:%s" % (topic_info, sql) return resp
def select(self, site, url_format, limit, start=0, extra_filter='{}', tube_name='download_rsp'): ret = RetStatus() first_result = self.select_handler.select_webpage( site, url_format, 1, start, extra_filter) if not first_result: ret.status = 0 return ret else: try: self.beanstalk.put(tube_name, self.to_string(first_result[0])) ret.status = 1 if limit > 1: # 创建守护进程,完成发送剩下数据 back_t = threading.Thread(target=self.back_work, args=(site, url_format, limit, start, extra_filter, tube_name)) back_t.setDaemon(True) back_t.start() return ret except Exception, e: log.error("put msg from:{0}\tresult:{1}".format( tube_name, str(e))) ret.status = 0 ret.errormessage = e.message return ret
def load_extractors(self, topic_id=-1): '''加载解析器''' mysql_db = get_mysql_conn(self.conf) if topic_id != -1: sql = '''select f_topic_id, f_module_path, f_module_name, f_extractor_name from extractors where f_topic_id=%s''' else: sql = '''select f_topic_id, f_module_path, f_module_name, f_extractor_name from extractors''' try: cursor = mysql_db.cursor() if topic_id != -1: param = [str(topic_id)] cursor.execute(sql, param) else: cursor.execute(sql) result = cursor.fetchall() mysql_db.close() for item in result: extractor_info = dbrecord_to_dict(cursor.description, item) topic_id = int(extractor_info.get("f_topic_id", -1)) self.modules_config[topic_id] = extractor_info except: log.error("query data from mysql error, sql:%s, ret:%s" % (sql, traceback.format_exc())) mysql_db.close() return None log.info("load_modules_success, moudle:%s" % json.dumps(self.modules_config)) return self.modules_config.keys()
def open_browser(url=None): set_headless = config.conf.get_str("env", "head_less") set_browser = config.conf.get_str("env", "browser") project_url = config.conf.get_str("env", "url") url = project_url if url is None else url try: if set_browser == "Chrome": options = webdriver.ChromeOptions() prefs = { "download.prompt_for_download": False, 'download.default_directory': config.downloads_dir, # 下载目录 'profile.default_content_settings.popups': 0, # 设置为0,禁止弹出窗口 'safebrowsing.enabled': True, } options.add_experimental_option('prefs', prefs) if set_headless == "True": options.add_argument('--headless') options.add_argument('--disable-gpu') driver = webdriver.Chrome(options=options, executable_path=config.driver_path) driver.maximize_window() driver.get(url=url) return driver else: raise Exception("暂不支持谷歌以外浏览器") except Exception as e: log.error("拉起浏览器失败") raise e
def pytest_runtest_makereport(item, call): # execute all other hooks to obtain the report object outcome = yield rep = outcome.get_result() # we only look at actual failing test calls, not setup/teardown if rep.when == "call" and not rep.passed: if 'global_test_env' in item.fixturenames: # download log in here try: log_name = item.funcargs["global_test_env"].backup_all_logs(item.name) job = item.funcargs["request"].config.getoption("--job") if job is None: log_url = os.path.join(item.funcargs["global_test_env"].cfg.bug_log, log_name) else: log_url = "http://{}:8080/job/PlatON/job/run/{}/artifact/logs/{}".format(socket.gethostbyname(socket.gethostname()), job, log_name) allure.attach('{}'.format(log_url), 'env log', allure.attachment_type.URI_LIST) except Exception as e: log.info("exception:{}".format(e)) # Record block number try: if item.funcargs["global_test_env"].running: env_status = "node blocks:{}".format(item.funcargs["global_test_env"].block_numbers()) else: env_status = "node runnings:{}".format(["{}:{}".format(node.node_mark, node.running) for node in item.funcargs["global_test_env"].get_all_nodes()]) log.info(env_status) allure.attach(env_status, "env status", allure.attachment_type.TEXT) except Exception as e: log.info("get block exception:{}".format(e)) else: log.error("This case does not use global_test_env")
def count_downloadsFiles(): try: count = len(os.listdir(config.downloads_dir)) log.info("获取下载目录文件数:{}".format(count)) return count except Exception as e: log.error("获取下载目录文件失败") raise e
def join(self): resp = self.to_main_queue.get(block=True) assert type(resp) == ResponsePacket if resp.is_succ(): self.process.join() log.debug("main proc : %s joined" % str(self.process.name)) else: log.error("Failed to stop subprocess " + str(self.process.name))
def do_js(self, locator, doc=""): ele = self.get_element(locator, doc=doc) try: self.driver.execute_script( 'arguments[0].removeAttribute("readonly")', ele) log.info('去除"' + locator[0] + '"只读属性') except Exception as e: log.error('去除"' + locator[0] + '"只读属性失败!') raise e
def __init(): cmd = '{} --datadir {} init {}'.format(self.remote_bin_file, self.remote_data_dir, self.remote_genesis_file) result = self.run_ssh(cmd) # todo :fix init complete # Adding a query here can only alleviate the problem of starting deployment without initialization. self.run_ssh("ls {}".format(self.remote_data_dir)) if len(result) > 0: log.error(failed_msg.format(self.node_mark, "init", result[0])) raise Exception("Init failed:{}".format(result[0])) log.debug("node-{} init success".format(self.node_mark))
def open_software(self, software_path, doc=""): try: # 启动软件 self.app = Application('uia').start(software_path) sleep(2) app2 = Application('uia').connect(path=software_path) dlg = app2['MicroSIP'] log.info(doc + "开启成功") except Exception as e: log.error(doc + "开启失败") raise e
def insert_extractor(self, extractor_info): '''往数据库中写入解析器''' resp = toolsutil.result() try: topic_id = extractor_info['topic_id'] target_dir_name = extractor_info['target_dir_name'] extractor_name = extractor_info.get('extractor_name', '') except: log.error("extractor_info error,\textractor_info:%s\tret:%s" % (extractor_info, traceback.format_exc())) resp['MSG'] = "extractor_info error,\textractor_info:%s\tret:%s" % (extractor_info, traceback.format_exc()) resp['CODE'] = -10000 return resp moudule_path = "extractors/" + target_dir_name + "/" + target_dir_name + "_extractor.py" if target_dir_name: moudule_name = target_dir_name[0].upper() + target_dir_name[1:] + "Extractor" else: moudule_name = "TemplateExtractor" # 1 创建解析器默认文件和默认代码 target_dir = self.basic_path + "i_entity_extractor/extractors/" + target_dir_name ret = self.copy_files(self.src_dir, target_dir) if not ret: resp['MSG'] = "copy_files_error,\textractor_info:%s\tret:%s" % (extractor_info, traceback.format_exc()) resp['CODE'] = -10000 return resp target_file = target_dir + "/" + target_dir_name + "_extractor.py" source_file = target_dir + "/template_extractor.py" open(target_file, "wb").write(open(source_file, "rb").read().replace("TemplateExtractor", moudule_name)) os.remove(source_file) # 2 在数据库中添加解析器配置 sql = '''INSERT INTO extractors (f_topic_id, f_module_path, f_module_name, f_extractor_name) VALUES(%s,%s,%s,%s)''' mysql_db = get_mysql_conn(self.conf) try: cursor = mysql_db.cursor() cursor.execute(sql, (topic_id, moudule_path, moudule_name, extractor_name)) except: resp['MSG'] = "insert data into mysql error,\textractor_info:%s\tsql:%s\tret:%s" % ( extractor_info, sql, traceback.format_exc()) resp['CODE'] = -10000 mysql_db.rollback() mysql_db.close() return resp mysql_db.commit() mysql_db.close() resp['MSG'] = "insert data into mysql success, extractor_info:%s, sql:%s" % (extractor_info, sql) return resp
def test_register_success(self, case): allure.dynamic() with allure.step("第一步:准备数据"): with allure.step("接口地址"): url = conf.get("env", "host") + case["url"] allure.attach( body=url, name="接口请求地址") # , extension=allure.attachment_type.TEXT with allure.step("请求方法"): method = case["method"] allure.attach(body=method, name="接口请求方法", extension=allure.attachment_type.TEXT) headers = eval(conf.get("env", "headers")) with allure.step("请求数据"): case["data"] = replace_data(case["data"]) json = case["data"] allure.attach(body=json, name="用例数据", extension=allure.attachment_type.JSON) with allure.step("预期结果"): expected = eval(case["expected"]) allure.attach(body=case["expected"], name="预期结果", extension=allure.attachment_type.TEXT) row = case["case_id"] + 1 # 发送请求 with allure.step("第二步:获取响应结果"): response = self.request.send_request(url=url, method=method, headers=headers, json=eval(json)) allure.attach(str(response.json()), "响应结果", allure.attachment_type.JSON) res = response.json() with allure.step("第三步:测试结果"): try: assert expected["code"] == res["code"] assert expected["msg"] == res["msg"] except AssertionError as e: self.excel1.write_data(row=None, column=8, value="未通过") log.error(f"用例{case['title']},测试未通过") log.exception(e) raise e else: self.excel1.write_data(row=row, column=8, value="通过") log.info(f"用例{case['title']},测试通过") allure.attach(f"{case['title']}用例测试通过", "测试结果", allure.attachment_type.TEXT)
def save_screenshot(self, doc=""): """ 截图 :param doc: :return: """ # 图片名称:模块名_页面名称_操作名称_年-月-日_时分秒.png file_path = config.screenshot_dir + \ "\\{0}_{1}.png".format(doc, time.strftime("%Y-%m-%d-%H-%M-%S")) try: self.driver.save_screenshot(file_path) log.info("截屏成功,图片路径为{}".format(file_path)) except Exception as e: log.error("截图失败") raise e
def start_handler(self, packet, from_main_queue, to_main_queue): try: self.on_start() self.input_thread = InputThread(conf=self.conf, proc_name=self.proc.process.name, processor=self.processor) self.input_thread.start() self.proc.running = True to_main_queue.put(ResponsePacket(packet, True)) except Exception as e: log.error(str(traceback.format_exc())) to_main_queue.put(ResponsePacket(packet, False)) raise e log.debug("%s : invoking start handler for process " % str(self.proc.process.name))
def clear_text(self, locator, doc=""): """ 清空输入内容 :param locator: 元素定位 :param doc: 模块名_页面名称_操作名称 :return: """ try: ele = self.get_element(locator) ele.clear() log.info('清空"' + locator[0] + '"文本') except Exception as e: log.error('清空"' + locator[0] + '"文本失败!') self.save_screenshot(doc) raise e
def get_element_attribute(self, locator, attr, doc=""): """ 获取元素属性 :param locator: 元素定位 :param attr: 属性名称 :param doc: 模块名_页面名称_操作名称 :return: """ ele = self.get_element(locator, doc=doc) try: return ele.get_attribute(attr) except Exception as e: log.error("获取元素属性失败!定位:{}".format(locator[0])) # 截图 self.save_screenshot(doc) raise e
def click_element(self, locator, doc=""): """ 点击操作 :param locator: 元素名称 :param doc: 模块名_页面名称_操作名称 :return: """ ele = self.get_element(locator, doc=doc) try: ele.click() log.info("点击:{0}".format(locator[0])) except Exception as e: log.error("点击" + locator[0] + "失败!") # 截图 self.save_screenshot(doc) raise e
def get_elements(self, locator, doc=""): """ 查找元素(多个) :param locator: 元素定位 :param doc: 模块名_页面名称_操作名称 :return: """ try: ele = self.driver.find_elements(*locator[1]) log.info("定位" + '"' + locator[0] + '"成功') return ele except Exception as e: log.error("定位" + '"' + locator[0] + '"失败') # 截图 self.save_screenshot(doc) raise e
def upload_file(file_path, file, doc='', *args): try: # 选择文件 app = pywinauto.Desktop() dlg = app["打开"] dlg["Toolbar3"].click() send_keys(file_path) send_keys("{VK_RETURN}") dlg["文件名(&N):Edit"].type_keys('"{}"'.format(file)) for i in args: send_keys('"{}"'.format(i)) dlg["打开(&O)"].click() log.info(doc + "上传文件成功") except Exception as e: log.error(doc + "上传文件失败") raise e
def init_connection_pool(conf): global pool_holder if pool_holder.pool is not None: # raise Exception("Trying to initialize mysql connection pool twice") return # Silently success try: pool_holder.pool = PersistentDB.PersistentDB(MySQLdb, 1000, host=conf['MYSQL']['host'], port=conf['MYSQL']['port'], user=conf['MYSQL']['username'], passwd=conf['MYSQL']['password'], db=conf['MYSQL']['dbname'], charset='utf8', setsession=['set autocommit = 1']) except Exception, e: log.error("failed to initialize mysql connection pool, message : %s" % (e.message)) log.error(traceback.format_exc())
def __init__(self, conf, processor=None, proc_name=None): threading.Thread.__init__(self) self.running = True self.proc_name = proc_name # Only for logging self.input_tube = conf['beanstalk_conf']['input_tube'] self.beanstalk = PyBeanstalk(conf['beanstalk_conf']['host'], conf['beanstalk_conf']['port']) self.out_beanstalk = PyBeanstalk(conf['beanstalk_conf']['host'], conf['beanstalk_conf']['port']) self.output_tube = conf['beanstalk_conf']['output_tube'] self.topic_output_tubes = {} self.topic_output_tubes.setdefault('default', []) """ output_tube = ["default_out", "only_special_out:1,2,3:exclusive", "special_out:4", ":5:exclusive"] topic_id:1,2,3只会用到only_special_out topic_id:4 会进入special_out和default_out topic_id:5 不会进入队列 topic_id:else 用用default_out队列 """ if type(self.output_tube) == list: for tube_def in self.output_tube: tube_def = tube_def.strip() if len(tube_def.split(":")) < 2: self.topic_output_tubes['default'].append(tube_def) else: elements = [a.strip() for a in tube_def.split(':')] tube_name = elements[0] topic_ids = [ int(a.strip()) for a in elements[1].split(',') ] exclusive = False if len(elements) == 3 and elements[2] == 'exclusive': exclusive = True for topic_id in topic_ids: self.topic_output_tubes.setdefault(topic_id, []) self.topic_output_tubes[topic_id].append( (tube_name, exclusive)) else: self.topic_output_tubes['default'].append(self.output_tube) self.log = log if processor is None: log.error("Processor not given !") raise Exception("Processor not given !") else: self.processor = processor
def get_text(self, locator, doc=""): """ 获取元素的文本内容 :param locator: 元素定位 :param doc: 模块名_页面名称_操作名称 :return: """ try: ele = self.get_element(locator, doc=doc) text = ele.text log.info('获取"' + locator[0] + '"文本,内容为:{}'.format(text)) return text except Exception as e: log.error('获取"' + locator[0] + '"文本内容失败!') # 截图 self.save_screenshot(doc) raise e
def input_text(self, locator, text, doc=""): """ 输入操作 :param locator: 元素定位 :param text: 输入的内容 :param doc: 模块名_页面名称_操作名称 :return: """ ele = self.get_element(locator, doc=doc) try: ele.send_keys(text) log.info('向"' + locator[0] + '"输入内容:{}'.format(text)) except Exception as e: log.error('向"' + locator[0] + '"输入内容:{}失败!'.format(text)) # 截图 self.save_screenshot(doc) raise e
def start(self): self.process.start() self.from_main_queue.put(StartPacket(), block=True) resp = self.to_main_queue.get(block=True) assert type(resp) == ResponsePacket if resp.is_succ(): log.debug("main proc : %s started successfully" % str(self.process.name)) self.running = True return True else: log.error("main proc : Failed to start subprocess " + str(self.process.name)) self.running = False #保证子进程启动不成功的is_alive状态为false self.process.join() return False