def getDetailData(self, url): data = {} data['detailUrl'] = url soup = SpiderUtil().getSoup(url) details = soup.find('div', class_='detail_xq w770') data['chname'] = details.h2.getText() for li in details.ul: if type(li) == bs4.element.Tag: texts = re.sub("(\t|\n|\r|\040)*", "", li.getText()).split(":") if texts[0] in Config().getCnnvdVulList(): codeName = Config().getCnnvdVulList()[texts[0]] data[codeName] = texts[1] print(codeName + ": " + data[codeName]) #漏洞简介 vul_descriptions = soup.find('div', class_='d_ldjj').findAll( 'p', style='text-indent:2em') data['vul_description'] = '' for vul_description in vul_descriptions: data['vul_description'] += re.sub("(\t|\n|\r|\040)*", "", vul_description.getText()) #漏洞公告,参考网址,受影响实体 contents = soup.findAll('div', class_='d_ldjj m_t_20') for content in contents: title = content.find('div', class_='title_bt').getText() title = re.sub("(\t|\n|\r|\040)*", "", title) if title in Config().getCnnvdVulList(): codeName = Config().getCnnvdVulList()[title] data[codeName] = '' p = content.findAll('p', style='text-indent:2em') for x in p: data[codeName] += re.sub("(\t|\n|\r|\040)*", "", x.getText()) return data
def getData(self, url): soup = SpiderUtil().getSoupByWebDriver(url) print(url) chname = soup.find("div", class_="blkContainerSblk").h1.getText() messageResult = {} messageResult['chname'] = chname tbody = soup.find("table", class_="gg_detail").tbody TRlist = tbody.find_all('tr') for trlist in TRlist[:-1]: if trlist.td.string == "影响产品": impact_productSum = '' if "影响产品" not in messageResult: messageResult["impact_product"] = [] for td in trlist.td.next_siblings: if type(td) == bs4.element.Tag: for k in td: impact_product = '' if type(k) == bs4.element.Tag: impact_product = re.sub( "(\t|\n|\r|\040)*", "", k.getText()) else: impact_product = re.sub( "(\t|\n|\r|\040)*", "", k.string) if impact_product != "": if impact_productSum == '': impact_productSum = impact_product else: impact_productSum = impact_productSum + ',' + impact_product messageResult['impact_product'].append(impact_productSum) else: name = trlist.td.string if name in Config().getCnvdVulList(): codename = Config().getCnvdVulList()[name] for td in trlist.td.next_siblings: if type(td) == bs4.element.Tag: tdText = re.sub(r"(\r|\t|\n|\040)*", "", td.getText()) if len(tdText): if codename in messageResult: messageResult[codename].append(tdText) else: messageResult[codename] = tdText else: self.logger.warning("url:{}, Chname:{}。 未收入的标签:{}".format( url, chname, name)) for name in Config().getCnvdVulList(): if Config().getCnvdVulList()[name] not in messageResult: messageResult[Config().getCnvdVulList()[name]] = NULL self.cnvdDao.insert(messageResult)
def __init__(self): super().__init__() self.qifu = StartServer() self.config = Config() self.worldId_gong = int(self.config.get_conf("worldId_gong")) self.worldId_pu1 = int(self.config.get_conf("worldId_pu1")) self.worldId_pu2 = int(self.config.get_conf("worldId_pu2")) self.areaId = int(self.config.get_conf("areaId")) self.versionId = int(self.config.get_conf("versionId")) self.groupname = time.strftime("%m.%d %H:%M:%S") self.time = (datetime.datetime.utcnow() + timedelta(hours=0.005)).isoformat() + 'Z'
def getSoup(self, url): req = request.Request(url, headers=Config().getHeader()) for i in range(Config().getMAX_NUM()): try: resp = request.urlopen(req) except urllib.error.URLError as e: if i < Config().getMAX_NUM() - 1: continue else: self.logger.error("{}:{}:次之后还是失败".format( url, Config().getMAX_NUM())) return content = resp.read() soup = BeautifulSoup(content, "lxml") return soup
def getDetail(self, url, id, type): detail = {} # 非编译器运行时 # path = os.getcwd() # executable_path = path + "\\chormedirver.exe" # print(executable_path) # browser = webdriver.Chrome(executable_path) browser = webdriver.Chrome() browser.get(url) categroySelect = Select(browser.find_element_by_id("categorySelect_0")) categroy = categroySelect.options[id].text if (categroy != '-请选择-'): detail['categroy'] = categroy categroySelect.select_by_visible_text(categroy.strip()) time.sleep(Config.getSleepTime()) brandSelect = Select(browser.find_element_by_id("brandSelect_0")) for brand in brandSelect.options: if (brand.text != '-请选择-'): #浏览器选择 detail['brand'] = brand.text brandSelect.select_by_visible_text(brand.text) time.sleep(Config.getSleepTime()) productSelect = Select( browser.find_element_by_id("ProductSelect_0")) for product in productSelect.options: if (product.text != '-请选择-'): detail['product'] = product.text #浏览器选择 productSelect.select_by_visible_text(product.text) time.sleep(Config.getSleepTime()) modelSelect = Select( browser.find_element_by_id("modelSelect_0")) # 存储全部选择 for model in modelSelect.options: if (model.text != '-请选择-'): detail['model'] = model.text # details.append(detail) print("类别: " + detail['categroy'] + " 品牌:" + detail['brand'] + " 系列:" + detail['product'] + " 型号:" + detail['model']) self.gongKongDao.insertDetail(detail, type) browser.close()
def getData(self, url): soup = SpiderUtil().getSoup(url) data = {} data['detail_url'] = url chname = soup.find('div', class_="page-header").getText() data["chname"] = re.sub("(\t|\n|\r|\040)*", "",chname) results = soup.findAll('div', class_="panel panel-success") for result in results: tag = [] for content in result.contents: if type(content) == bs4.Tag: tag.append(content) if len(tag) == 2: if tag[0].getText() == "漏洞参数": for p in tag[1].contents: if type(p) == bs4.Tag: text = p.getText().split(":") if len(text) > 0 and text[0] in Config().getIVDVulList(): data[Config().getIVDVulList()[text[0]]] = text[1] elif tag[0].getText() in Config().getIVDVulList(): data[Config().getIVDVulList()[tag[0].getText()]] = re.sub("(\t|\n|\r|\040)*", "", tag[1].getText()) self.ivdDao.insert(data)
def xx(self, dataList): dataCount = {} pattern = re.compile(Config().getFilterFile2()) for data in dataList: datas = data.url.split("/") for key in datas: match = pattern.match(key) if match == None and key != '': if not dataCount.get(key): dataCount[key] = 1 else: dataCount[key] += 1 sort = sorted(dataCount.items(), key=lambda e: e[1], reverse=True) for item in sort: print(item) self.logger.debug(item)
def getUrlsByGoogle(self, keyWord): datas = [] # 对url中中文处理 start_url = "https://www.google.com.hk/search?q=" + urllib.parse.quote( keyWord) browser = webdriver.Chrome() browser.get(start_url) while (self.isElementExist(browser, '下一页')): browser.find_element_by_link_text('下一页').click() soup = BeautifulSoup(browser.page_source, "html.parser", from_encoding='UTF-8') self.getDataByGoogle(soup, keyWord, datas) time.sleep(Config().getSleepTime()) browser.close() self.urlKeyWordDao.insert(datas)
def load_logging_config(): current_dir = os.path.dirname(os.path.realpath(__file__)) log_config_path = os.path.join(current_dir, 'logging.yaml') logs_dir = Config().get('logs_dir') try: with open(log_config_path, 'rt') as f: timestamp = datetime.now().strftime("%d.%m.%Y.%H.%M.%S") filepath = os.path.join(logs_dir, f'{timestamp}.log') config = yaml.safe_load(f.read()) config['handlers']['empty_file_handler']['filename'] = filepath config['handlers']['pretty_file_handler']['filename'] = filepath except yaml.YAMLError as e: raise Exception(f"Failed to parse '{log_config_path}'") from e logging.config.dictConfig(config)
def dataPreparation(self): dataCount = {} dataList = self.urlKeyWordDao.listURL() pattern = re.compile(Config().getFilterFile()) for data in dataList: datas = data.url.split(".") for key in datas: match = pattern.match(key) if match == None and key != '': if not dataCount.get(key): dataCount[key] = 1 else: dataCount[key] += 1 sort = sorted(dataCount.items(), key=lambda e: e[1], reverse=True) for item in sort: print(item) self.logger.debug(item)
class TestOpen: def __init__(self): self.config = Config() self.url = self.config.get_conf('url') + "api/task/open-server" def test_open_gong(self, data): # 开公共区 response = SendMethodEntity.send_method("post", self.url, data=data) task_id = response["data"]["taskIds"][0] GetTask.test_code(task_id) return response def test_open_pu(self, data): # 开普通区 response = SendMethodEntity.send_method("post", self.url, data=data) task_id = response["data"]["taskIds"][0] GetTask.test_code(task_id) return response
def __init__(self): self.header = Config().getHeader() self.logger = MyLog().getLogger() self.gongKongDao = GongKongDao()
def __init__(self): self.header = Config().getHeader() self.logger = MyLog().getLogger() self.urlKeyWordDao = UrlKeyWordDao()
def __init__(self): self.header = Config().getHeader() self.logger = MyLog().getLogger() self.cnnvdDao = CnnvdDao()
class BasePage: """Base class to initialize the base page that will be called from all pages""" url = None _page_identifier_element = None _page_load_timeout = Config().get('page_load_timeout') _element_wait_timeout = Config().get('element_wait_timeout') log = logging.getLogger('pretty_logger') def __init__(self, driver): self.driver = driver self.verify_page_is_loaded() # add logging to all methods # def __getattribute__(self, name): # returned = object.__getattribute__(self, name) # if inspect.isfunction(returned) or inspect.ismethod(returned): # self.log.info(f"{self.__class__.__name__}: called {returned.__name__}") # return returned def verify_page_is_loaded(self): try: self._is_page_identifier_element_displayed() self._is_document_in_ready_state() except TimeoutException: message = f"{self.__class__.__name__} was not loaded." self.log.exception(message) # TODO this exception is not captured in logs raise PageNotLoadedException(message) def _is_page_identifier_element_displayed(self): """Wait until self._page_identifier element is displayed""" try: self.wait_until(EC.visibility_of_element_located( self._page_identifier_element), timeout=self._element_wait_timeout) self.log.info( f"{self.__class__.__name__}: Page identifier {self._page_identifier_element} was displayed." ) except TimeoutException: raise TimeoutException( f"'{self.__class__.__name__}': Page identifier element {self._page_identifier_element} was not visible during {self._element_wait_timeout} seconds." ) def _is_document_in_ready_state(self): self.wait_until(document_has_ready_state()) self.log.info( f"{self.__class__.__name__}: Document was in ready state.") def is_loaded(self): # TODO raises NoSuchElementException when cannot find element, should I handle and return False for not displayed - that may hide the fact that element is not present return self.driver.find_element( *self._page_identifier_element).is_displayed() def is_not_loaded(self): return not self.driver.find_element( *self._page_identifier_element).is_displayed() def get_url(self): return self.driver.current_url # region Wait methods def wait_until(self, condition_method, timeout=5, **kwargs): """wait for the event in condition_method, log TimeoutException and re-raise""" wait = WebDriverWait(self.driver, timeout, **kwargs) try: return wait.until(condition_method) except TimeoutException: raise TimeoutException( f"Event '{condition_method.__class__.__name__}' did not occur during '{timeout}' seconds. Vars: {vars(condition_method)}" ) def wait_until_displayed(self, locator, timeout=5, **kwargs): try: return self.wait_until(EC.visibility_of_element_located(locator), timeout, **kwargs) except TimeoutException: raise ElementNotDisplayedException( f"{self.__class__.__name__} page: Element {locator} was not displayed during {timeout} seconds." ) def wait_until_clickable(self, locator, timeout=5, **kwargs): try: return self.wait_until(EC.element_to_be_clickable(locator), timeout, **kwargs) except TimeoutException: raise ElementNotClickableException( f"{self.__class__.__name__} page: Element {locator} was not clickable during {timeout} seconds." ) # endregion def _is_element_displayed(self, locator): try: return self.driver.find_element(*locator).is_displayed() except NoSuchElementException: return False # region driver public API def get_element(self, locator): try: element = self.driver.find_element(*locator) self.log.info( f"{self.__class__.__name__}: Element with locator {locator} was found." ) return element except NoSuchElementException: self.log.exception( f"{self.__class__.__name__}: Element with locator {locator} was not found." ) raise def enter_text(self, locator, text): self.get_element(locator).send_keys(text) self.log.info( f"{self.__class__.__name__}: Entered text '{text}' into element with locator {locator}." ) def click_element(self, locator): self.get_element(locator).click() self.log.info( f"{self.__class__.__name__}: Clicked on element with locator {locator}." ) def click_element_with_JS(self, locator): element = self.get_element(locator) self.driver.execute_script("arguments[0].click();", element) self.log.info( f"{self.__class__.__name__}: Clicked with JS on element with locator {locator}." ) def get_text(self, locator): element_text = self.get_element(locator).text self.log.info( f"{self.__class__.__name__}: Element with locator {locator} has text '{element_text}'." ) return element_text def get_element_attribute(self, locator, attribute): element_attr = self.get_element(locator).get_attribute(attribute) self.log.info( f"{self.__class__.__name__}: Element with locator {locator} has attribute '{attribute}' with value '{element_attr}'." ) return element_attr def get_element_value(self, locator): element_value = self.get_element(locator).get_attribute('value') self.log.info( f"{self.__class__.__name__}: Element with locator {locator} has value '{element_value}'." ) return element_value def switch_to_new_tab(self): new_opened_tab_handle = self.driver.window_handles[1] self.driver.switch_to.window(new_opened_tab_handle) self.log.info( f"{self.__class__.__name__}: Switched to new tab with handle {new_opened_tab_handle}." ) def select_dropdown_option(self, locator, option_name): Select(self.get_element(locator)).select_by_visible_text(option_name) self.log.info( f"{self.__class__.__name__}: Selected drop-down option '{option_name}'." ) def mouse_over(self, locator): element = self.get_element(locator) ActionChains(self.driver).move_to_element(element).perform() self.log.info( f"{self.__class__.__name__}: Hovered moused over element with locator '{locator}'." )
def __init__(self, session): self.session = session self.config = Config() self.game_id = self.config.game_id self.region_id = self.config.region_id
def config(): return Config()
from selenium.webdriver.chrome.options import Options as ChromeOptions from selenium.webdriver.firefox.options import Options as FirefoxOptions from common.logger.custom_logger import load_logging_config from common.data.credentials import get_credentials from common.config.config import Config from models.user import User from pages.login_page import LoginPage from pages.header_page import Header from datetime import datetime from py.xml import html from pages.menu.main_menu import MainMenu RESULTS_DIR = Config().get('results_dir') SCREENSHOTS_DIR = Config().get('screenshots_dir') LOGS_DIR = Config().get('logs_dir') @pytest.fixture(scope="session", autouse=True) def create_results_dirs(config): Path(RESULTS_DIR).mkdir(parents=True, exist_ok=True) Path(SCREENSHOTS_DIR).mkdir(parents=True, exist_ok=True) Path(LOGS_DIR).mkdir(parents=True, exist_ok=True) load_logging_config() empty_logger = logging.getLogger('empty_logger') pretty_logger = logging.getLogger('pretty_logger')
class TestStart(Config): def __init__(self): super().__init__() self.qifu = StartServer() self.config = Config() self.worldId_gong = int(self.config.get_conf("worldId_gong")) self.worldId_pu1 = int(self.config.get_conf("worldId_pu1")) self.worldId_pu2 = int(self.config.get_conf("worldId_pu2")) self.areaId = int(self.config.get_conf("areaId")) self.versionId = int(self.config.get_conf("versionId")) self.groupname = time.strftime("%m.%d %H:%M:%S") self.time = (datetime.datetime.utcnow() + timedelta(hours=0.005)).isoformat() + 'Z' def test_serverlist_add(self): # serverlist修改完成后,对新区进行对外 data = { "gameId": self.game_id, "regionId": self.region_id, "areaId": None, "serverListId": 11, "data": [{ "opsType": 4, "row": { "name": self.groupname, "channelId": 99, "sort": 1 }, "info": { "name": None } }], "opsType": 1 } response = self.qifu.test_serverlist_add(data) grouplist = self.qifu.test_groupid( self.groupname)['data'][1]['groupList'] group = next(x for x in grouplist if x['name'] == self.groupname) groupid = group["id"] time.sleep(2) data = { "rangeData": [{ "areaId": self.areaId, "worlds": [self.worldId_gong, self.worldId_pu1, self.worldId_pu2] }], "extraIds": [], "gameId": self.game_id, "regionId": self.region_id, "executeTime": self.time, "serverList": [{ "worldId": self.worldId_pu1, "channelId": 21, "groupId": groupid, "recommend": 1, "channelName": "自动化测试", "groupName": self.groupname, "worldName": str(self.worldId_pu1) + "-自动化-普通区-1" }, { "worldId": self.worldId_pu2, "channelId": 21, "groupId": groupid, "recommend": 1, "channelName": "自动化测试", "groupName": self.groupname, "worldName": str(self.worldId_pu2) + "-自动化-普通区-2" }] } response = self.qifu.test_foreign(data) return response def test_foreign(self): # 对外 data = { "rangeData": [{ "areaId": self.areaId, "worlds": [self.worldId_gong, self.worldId_pu1] }], "extraIds": [], "gameId": self.game_id, "regionId": self.region_id, "executeTime": self.time, "serverList": [], "serverRecommend": [], "channelRecommend": {} } response = self.qifu.test_foreign(data) return response def test_script(self): # 刷脚本 data = { "range": 3, "executeTime": None, "rangeData": [{ "areaId": self.areaId, "worlds": [self.worldId_gong, self.worldId_pu1, self.worldId_pu2] }], "gameId": self.game_id, "regionId": self.region_id, "fileList": [{ "sort": 1, "repairFunction": "测试", "fileName": "[Social]RefreshKeyWords.lua", "uid": "", "fileId": 73, "restartRepair": 0, "newRepair": 0, "operationType": 1, "serverType": 8 }], "includeProtected": 1 } response = self.qifu.test_script(data) return response def test_stop(self): # 关服 data = { "rangeData": [{ "areaId": self.areaId, "worlds": [self.worldId_gong, self.worldId_pu1, self.worldId_pu2] }], "extraIds": [], "gameId": self.game_id, "regionId": self.region_id, "isForceKill": 1, "executeTime": self.time } response = self.qifu.test_stop(data) return response def test_start(self): # 启服 data = { "rangeData": [{ "areaId": self.areaId, "worlds": [self.worldId_gong, self.worldId_pu1, self.worldId_pu2] }], "extraIds": [], "gameId": self.game_id, "regionId": self.region_id, "executeTime": self.time, "specId": None, "specName": None, "versionId": None, "versionName": None, "isFileupdate": 1 } # data = { # "rangeData": [ # { # "areaId": self.areaId, # "worlds": [ # self.worldId_gong, # self.worldId_pu1, # self.worldId_pu2 # ] # } # ], # "extraIds": [], # "gameId": self.game_id, # "regionId": self.region_id, # "executeTime": self.time, # "specList": [ # { # "specsId": self.specsId_pu, # "worldId": self.worldId_pu1, # "areaId": 13, # "groupIndex": 0, # "f_spec": { # "id": 96, # "name": "远征2普通区-1S-ECS" # } # }, # { # "specsId": 98, # "specsName": "远征2公共区-2S-ECS", # "worldId": 3523, # "worldName": "0806-公共一区-V2.2", # "areaId": 13, # "areaName": "自动化运维预演", # "groupIndex": 1, # "f_spec": { # "id": 98, # "name": "远征2公共区-2S-ECS" # } # } # ], # "versionId": 288, # "versionName": "2021.06.03-HWAutoDel", # "isFileupdate": 1 # } response = self.qifu.test_start(data) return response def test_clear(self): # 新区清数据 data = { "rangeData": [{ "areaId": self.areaId, "worlds": [self.worldId_pu1, self.worldId_pu2] }], "extraIds": [], "gameId": self.game_id, "regionId": self.region_id, "executeTime": self.time, "isBook": 0, "preCreateTime": None, "openTime": (datetime.datetime.utcnow() + datetime.timedelta(hours=24)).isoformat() + 'Z', "isForeign": 0, "foreignTime": None, "serverList": [] } response = self.qifu.test_clear(data) return response def test_container_upload(self): # 文件上传 filePath = self.qifu.test_() data = { "rangeData": [{ "areaId": self.areaId, "worlds": [self.worldId_pu1] }], "extraIds": [], "gameId": self.game_id, "regionId": self.region_id, "fileList": [{ "id": 0, "file": None, "fileName": "ChatControl.csv", "filePath": filePath, "uploadPath": "C:\\app\\cluster\\data\\global\\scp" }] } response = self.qifu.test_container_upload(data) return response def test_stop_operate(self): # 停运 data = { "rangeData": [{ "areaId": self.areaId, "worlds": [self.worldId_gong, self.worldId_pu1] }], "extraIds": [], "gameId": self.game_id, "regionId": self.region_id, "executeTime": self.time } response = self.qifu.test_stop_operate(data) # 停运完成后增加区服ID。写入到config worldId_gong = str(int(self.config.worldId_gong) + 1) worldId_pu1 = str(int(self.config.worldId_pu1) + 1) worldId_pu2 = str(int(self.config.worldId_pu2) + 1) self.config.set_conf(self.config.WORLDID_GONG, worldId_gong) self.config.set_conf(self.config.WORLDID_PU1, worldId_pu1) self.config.set_conf(self.config.WORLDID_PU2, worldId_pu2) return response
class GetTask(object): def __init__(self, taskId): self.config = Config() self.url = self.config.get_conf('url') self.taskId = taskId self.ins = '' def test_check(self): # 对提交的任务进行审核 url = self.url + f"api/task/{self.taskId}/approve" data = {"remark": "接口执行通过", "status": 0} response = SendMethodEntity.send_method("post", url, data=data) logging.info(response) return response def test_task_revocation(self): # 对任务进行驳回 url = self.url + f"api/task/{self.taskId}/approve" data = {"remark": "接口执行驳回", "status": 1} response = SendMethodEntity.send_method("post", url, data=data) logging.info(response) return response def test_task_particulars(self): # 查看任务详情 count = 0 while True: url = self.url + f"api/task/{self.taskId}/detail" response = SendMethodEntity.send_method( "get", url, ) status = response["data"]["status"] logging.info(response) if status == 7 or status == 4: logging.info(response) break if status == 8 or status == 30 or status == 20: # 失败后对任务进行重试,重试三次后跳出循环 if count >= 3: raise ValueError("重试失败") count += 1 response = SendMethodEntity.send_method( "post", self.url + f"api/task/{self.taskId}/retry-skip", data={"action": 1}) time.sleep(30) def test_reject(self): url = url = self.url + f"api/task/{self.taskId}/approve" pass @staticmethod def test_code(task_id): Get_Task = GetTask(task_id) Get_Task.test_check() Get_Task.test_task_particulars() @staticmethod def test_revocation(task_id): Get_revocation = GetTask(task_id) Get_revocation.test_task_revocation() Get_revocation.test_task_particulars()
def __init__(self, taskId): self.config = Config() self.url = self.config.get_conf('url') self.taskId = taskId self.ins = ''
def __init__(self): self.config = Config() self.url = self.config.get_conf('url') + "api/task/open-server"