def getDetailData(self, url): data = {} data['detailUrl'] = url soup = SpiderUtil().getSoup(url) details = soup.find('div', class_='detail_xq w770') data['chname'] = details.h2.getText() for li in details.ul: if type(li) == bs4.element.Tag: texts = re.sub("(\t|\n|\r|\040)*", "", li.getText()).split(":") if texts[0] in Config().getCnnvdVulList(): codeName = Config().getCnnvdVulList()[texts[0]] data[codeName] = texts[1] print(codeName + ": " + data[codeName]) #漏洞简介 vul_descriptions = soup.find('div', class_='d_ldjj').findAll( 'p', style='text-indent:2em') data['vul_description'] = '' for vul_description in vul_descriptions: data['vul_description'] += re.sub("(\t|\n|\r|\040)*", "", vul_description.getText()) #漏洞公告,参考网址,受影响实体 contents = soup.findAll('div', class_='d_ldjj m_t_20') for content in contents: title = content.find('div', class_='title_bt').getText() title = re.sub("(\t|\n|\r|\040)*", "", title) if title in Config().getCnnvdVulList(): codeName = Config().getCnnvdVulList()[title] data[codeName] = '' p = content.findAll('p', style='text-indent:2em') for x in p: data[codeName] += re.sub("(\t|\n|\r|\040)*", "", x.getText()) return data
def getData(self, url): soup = SpiderUtil().getSoupByWebDriver(url) print(url) chname = soup.find("div", class_="blkContainerSblk").h1.getText() messageResult = {} messageResult['chname'] = chname tbody = soup.find("table", class_="gg_detail").tbody TRlist = tbody.find_all('tr') for trlist in TRlist[:-1]: if trlist.td.string == "影响产品": impact_productSum = '' if "影响产品" not in messageResult: messageResult["impact_product"] = [] for td in trlist.td.next_siblings: if type(td) == bs4.element.Tag: for k in td: impact_product = '' if type(k) == bs4.element.Tag: impact_product = re.sub( "(\t|\n|\r|\040)*", "", k.getText()) else: impact_product = re.sub( "(\t|\n|\r|\040)*", "", k.string) if impact_product != "": if impact_productSum == '': impact_productSum = impact_product else: impact_productSum = impact_productSum + ',' + impact_product messageResult['impact_product'].append(impact_productSum) else: name = trlist.td.string if name in Config().getCnvdVulList(): codename = Config().getCnvdVulList()[name] for td in trlist.td.next_siblings: if type(td) == bs4.element.Tag: tdText = re.sub(r"(\r|\t|\n|\040)*", "", td.getText()) if len(tdText): if codename in messageResult: messageResult[codename].append(tdText) else: messageResult[codename] = tdText else: self.logger.warning("url:{}, Chname:{}。 未收入的标签:{}".format( url, chname, name)) for name in Config().getCnvdVulList(): if Config().getCnvdVulList()[name] not in messageResult: messageResult[Config().getCnvdVulList()[name]] = NULL self.cnvdDao.insert(messageResult)
def getSoup(self, url): req = request.Request(url, headers=Config().getHeader()) for i in range(Config().getMAX_NUM()): try: resp = request.urlopen(req) except urllib.error.URLError as e: if i < Config().getMAX_NUM() - 1: continue else: self.logger.error("{}:{}:次之后还是失败".format( url, Config().getMAX_NUM())) return content = resp.read() soup = BeautifulSoup(content, "lxml") return soup
def __init__(self): super().__init__() self.qifu = StartServer() self.config = Config() self.worldId_gong = int(self.config.get_conf("worldId_gong")) self.worldId_pu1 = int(self.config.get_conf("worldId_pu1")) self.worldId_pu2 = int(self.config.get_conf("worldId_pu2")) self.areaId = int(self.config.get_conf("areaId")) self.versionId = int(self.config.get_conf("versionId")) self.groupname = time.strftime("%m.%d %H:%M:%S") self.time = (datetime.datetime.utcnow() + timedelta(hours=0.005)).isoformat() + 'Z'
def getData(self, url): soup = SpiderUtil().getSoup(url) data = {} data['detail_url'] = url chname = soup.find('div', class_="page-header").getText() data["chname"] = re.sub("(\t|\n|\r|\040)*", "",chname) results = soup.findAll('div', class_="panel panel-success") for result in results: tag = [] for content in result.contents: if type(content) == bs4.Tag: tag.append(content) if len(tag) == 2: if tag[0].getText() == "漏洞参数": for p in tag[1].contents: if type(p) == bs4.Tag: text = p.getText().split(":") if len(text) > 0 and text[0] in Config().getIVDVulList(): data[Config().getIVDVulList()[text[0]]] = text[1] elif tag[0].getText() in Config().getIVDVulList(): data[Config().getIVDVulList()[tag[0].getText()]] = re.sub("(\t|\n|\r|\040)*", "", tag[1].getText()) self.ivdDao.insert(data)
def getUrlsByGoogle(self, keyWord): datas = [] # 对url中中文处理 start_url = "https://www.google.com.hk/search?q=" + urllib.parse.quote( keyWord) browser = webdriver.Chrome() browser.get(start_url) while (self.isElementExist(browser, '下一页')): browser.find_element_by_link_text('下一页').click() soup = BeautifulSoup(browser.page_source, "html.parser", from_encoding='UTF-8') self.getDataByGoogle(soup, keyWord, datas) time.sleep(Config().getSleepTime()) browser.close() self.urlKeyWordDao.insert(datas)
def load_logging_config(): current_dir = os.path.dirname(os.path.realpath(__file__)) log_config_path = os.path.join(current_dir, 'logging.yaml') logs_dir = Config().get('logs_dir') try: with open(log_config_path, 'rt') as f: timestamp = datetime.now().strftime("%d.%m.%Y.%H.%M.%S") filepath = os.path.join(logs_dir, f'{timestamp}.log') config = yaml.safe_load(f.read()) config['handlers']['empty_file_handler']['filename'] = filepath config['handlers']['pretty_file_handler']['filename'] = filepath except yaml.YAMLError as e: raise Exception(f"Failed to parse '{log_config_path}'") from e logging.config.dictConfig(config)
def xx(self, dataList): dataCount = {} pattern = re.compile(Config().getFilterFile2()) for data in dataList: datas = data.url.split("/") for key in datas: match = pattern.match(key) if match == None and key != '': if not dataCount.get(key): dataCount[key] = 1 else: dataCount[key] += 1 sort = sorted(dataCount.items(), key=lambda e: e[1], reverse=True) for item in sort: print(item) self.logger.debug(item)
def dataPreparation(self): dataCount = {} dataList = self.urlKeyWordDao.listURL() pattern = re.compile(Config().getFilterFile()) for data in dataList: datas = data.url.split(".") for key in datas: match = pattern.match(key) if match == None and key != '': if not dataCount.get(key): dataCount[key] = 1 else: dataCount[key] += 1 sort = sorted(dataCount.items(), key=lambda e: e[1], reverse=True) for item in sort: print(item) self.logger.debug(item)
def __init__(self, taskId): self.config = Config() self.url = self.config.get_conf('url') self.taskId = taskId self.ins = ''
def __init__(self): self.header = Config().getHeader() self.logger = MyLog().getLogger() self.cnnvdDao = CnnvdDao()
def __init__(self): self.header = Config().getHeader() self.logger = MyLog().getLogger() self.gongKongDao = GongKongDao()
def __init__(self): self.header = Config().getHeader() self.logger = MyLog().getLogger() self.urlKeyWordDao = UrlKeyWordDao()
def __init__(self): self.config = Config() self.url = self.config.get_conf('url') + "api/task/open-server"
class BasePage: """Base class to initialize the base page that will be called from all pages""" url = None _page_identifier_element = None _page_load_timeout = Config().get('page_load_timeout') _element_wait_timeout = Config().get('element_wait_timeout') log = logging.getLogger('pretty_logger') def __init__(self, driver): self.driver = driver self.verify_page_is_loaded() # add logging to all methods # def __getattribute__(self, name): # returned = object.__getattribute__(self, name) # if inspect.isfunction(returned) or inspect.ismethod(returned): # self.log.info(f"{self.__class__.__name__}: called {returned.__name__}") # return returned def verify_page_is_loaded(self): try: self._is_page_identifier_element_displayed() self._is_document_in_ready_state() except TimeoutException: message = f"{self.__class__.__name__} was not loaded." self.log.exception(message) # TODO this exception is not captured in logs raise PageNotLoadedException(message) def _is_page_identifier_element_displayed(self): """Wait until self._page_identifier element is displayed""" try: self.wait_until(EC.visibility_of_element_located( self._page_identifier_element), timeout=self._element_wait_timeout) self.log.info( f"{self.__class__.__name__}: Page identifier {self._page_identifier_element} was displayed." ) except TimeoutException: raise TimeoutException( f"'{self.__class__.__name__}': Page identifier element {self._page_identifier_element} was not visible during {self._element_wait_timeout} seconds." ) def _is_document_in_ready_state(self): self.wait_until(document_has_ready_state()) self.log.info( f"{self.__class__.__name__}: Document was in ready state.") def is_loaded(self): # TODO raises NoSuchElementException when cannot find element, should I handle and return False for not displayed - that may hide the fact that element is not present return self.driver.find_element( *self._page_identifier_element).is_displayed() def is_not_loaded(self): return not self.driver.find_element( *self._page_identifier_element).is_displayed() def get_url(self): return self.driver.current_url # region Wait methods def wait_until(self, condition_method, timeout=5, **kwargs): """wait for the event in condition_method, log TimeoutException and re-raise""" wait = WebDriverWait(self.driver, timeout, **kwargs) try: return wait.until(condition_method) except TimeoutException: raise TimeoutException( f"Event '{condition_method.__class__.__name__}' did not occur during '{timeout}' seconds. Vars: {vars(condition_method)}" ) def wait_until_displayed(self, locator, timeout=5, **kwargs): try: return self.wait_until(EC.visibility_of_element_located(locator), timeout, **kwargs) except TimeoutException: raise ElementNotDisplayedException( f"{self.__class__.__name__} page: Element {locator} was not displayed during {timeout} seconds." ) def wait_until_clickable(self, locator, timeout=5, **kwargs): try: return self.wait_until(EC.element_to_be_clickable(locator), timeout, **kwargs) except TimeoutException: raise ElementNotClickableException( f"{self.__class__.__name__} page: Element {locator} was not clickable during {timeout} seconds." ) # endregion def _is_element_displayed(self, locator): try: return self.driver.find_element(*locator).is_displayed() except NoSuchElementException: return False # region driver public API def get_element(self, locator): try: element = self.driver.find_element(*locator) self.log.info( f"{self.__class__.__name__}: Element with locator {locator} was found." ) return element except NoSuchElementException: self.log.exception( f"{self.__class__.__name__}: Element with locator {locator} was not found." ) raise def enter_text(self, locator, text): self.get_element(locator).send_keys(text) self.log.info( f"{self.__class__.__name__}: Entered text '{text}' into element with locator {locator}." ) def click_element(self, locator): self.get_element(locator).click() self.log.info( f"{self.__class__.__name__}: Clicked on element with locator {locator}." ) def click_element_with_JS(self, locator): element = self.get_element(locator) self.driver.execute_script("arguments[0].click();", element) self.log.info( f"{self.__class__.__name__}: Clicked with JS on element with locator {locator}." ) def get_text(self, locator): element_text = self.get_element(locator).text self.log.info( f"{self.__class__.__name__}: Element with locator {locator} has text '{element_text}'." ) return element_text def get_element_attribute(self, locator, attribute): element_attr = self.get_element(locator).get_attribute(attribute) self.log.info( f"{self.__class__.__name__}: Element with locator {locator} has attribute '{attribute}' with value '{element_attr}'." ) return element_attr def get_element_value(self, locator): element_value = self.get_element(locator).get_attribute('value') self.log.info( f"{self.__class__.__name__}: Element with locator {locator} has value '{element_value}'." ) return element_value def switch_to_new_tab(self): new_opened_tab_handle = self.driver.window_handles[1] self.driver.switch_to.window(new_opened_tab_handle) self.log.info( f"{self.__class__.__name__}: Switched to new tab with handle {new_opened_tab_handle}." ) def select_dropdown_option(self, locator, option_name): Select(self.get_element(locator)).select_by_visible_text(option_name) self.log.info( f"{self.__class__.__name__}: Selected drop-down option '{option_name}'." ) def mouse_over(self, locator): element = self.get_element(locator) ActionChains(self.driver).move_to_element(element).perform() self.log.info( f"{self.__class__.__name__}: Hovered moused over element with locator '{locator}'." )
def __init__(self, session): self.session = session self.config = Config() self.game_id = self.config.game_id self.region_id = self.config.region_id
from selenium.webdriver.chrome.options import Options as ChromeOptions from selenium.webdriver.firefox.options import Options as FirefoxOptions from common.logger.custom_logger import load_logging_config from common.data.credentials import get_credentials from common.config.config import Config from models.user import User from pages.login_page import LoginPage from pages.header_page import Header from datetime import datetime from py.xml import html from pages.menu.main_menu import MainMenu RESULTS_DIR = Config().get('results_dir') SCREENSHOTS_DIR = Config().get('screenshots_dir') LOGS_DIR = Config().get('logs_dir') @pytest.fixture(scope="session", autouse=True) def create_results_dirs(config): Path(RESULTS_DIR).mkdir(parents=True, exist_ok=True) Path(SCREENSHOTS_DIR).mkdir(parents=True, exist_ok=True) Path(LOGS_DIR).mkdir(parents=True, exist_ok=True) load_logging_config() empty_logger = logging.getLogger('empty_logger') pretty_logger = logging.getLogger('pretty_logger')
def config(): return Config()