def getDetailData(self, url):
data = {}
data['detailUrl'] = url
soup = SpiderUtil().getSoup(url)
details = soup.find('div', class_='detail_xq w770')
data['chname'] = details.h2.getText()
for li in details.ul:
if type(li) == bs4.element.Tag:
texts = re.sub("(\t|\n|\r|\040)*", "", li.getText()).split("：")
if texts[0] in Config().getCnnvdVulList():
codeName = Config().getCnnvdVulList()[texts[0]]
data[codeName] = texts[1]
print(codeName + ": " + data[codeName])
#漏洞简介
vul_descriptions = soup.find('div', class_='d_ldjj').findAll(
'p', style='text-indent:2em')
data['vul_description'] = ''
for vul_description in vul_descriptions:
data['vul_description'] += re.sub("(\t|\n|\r|\040)*", "",
vul_description.getText())
#漏洞公告，参考网址，受影响实体
contents = soup.findAll('div', class_='d_ldjj m_t_20')
for content in contents:
title = content.find('div', class_='title_bt').getText()
title = re.sub("(\t|\n|\r|\040)*", "", title)
if title in Config().getCnnvdVulList():
codeName = Config().getCnnvdVulList()[title]
data[codeName] = ''
p = content.findAll('p', style='text-indent:2em')
for x in p:
data[codeName] += re.sub("(\t|\n|\r|\040)*", "",
x.getText())
return data
def getData(self, url):
soup = SpiderUtil().getSoupByWebDriver(url)
print(url)
chname = soup.find("div", class_="blkContainerSblk").h1.getText()
messageResult = {}
messageResult['chname'] = chname
tbody = soup.find("table", class_="gg_detail").tbody
TRlist = tbody.find_all('tr')
for trlist in TRlist[:-1]:
if trlist.td.string == "影响产品":
impact_productSum = ''
if "影响产品" not in messageResult:
messageResult["impact_product"] = []
for td in trlist.td.next_siblings:
if type(td) == bs4.element.Tag:
for k in td:
impact_product = ''
if type(k) == bs4.element.Tag:
impact_product = re.sub(
"(\t|\n|\r|\040)*", "", k.getText())
else:
impact_product = re.sub(
"(\t|\n|\r|\040)*", "", k.string)
if impact_product != "":
if impact_productSum == '':
impact_productSum = impact_product
else:
impact_productSum = impact_productSum + ',' + impact_product
messageResult['impact_product'].append(impact_productSum)
else:
name = trlist.td.string
if name in Config().getCnvdVulList():
codename = Config().getCnvdVulList()[name]
for td in trlist.td.next_siblings:
if type(td) == bs4.element.Tag:
tdText = re.sub(r"(\r|\t|\n|\040)*", "",
td.getText())
if len(tdText):
if codename in messageResult:
messageResult[codename].append(tdText)
else:
messageResult[codename] = tdText
else:
self.logger.warning("url:{}, Chname:{}。 未收入的标签：{}".format(
url, chname, name))
for name in Config().getCnvdVulList():
if Config().getCnvdVulList()[name] not in messageResult:
messageResult[Config().getCnvdVulList()[name]] = NULL
self.cnvdDao.insert(messageResult)
def getSoup(self, url):
req = request.Request(url, headers=Config().getHeader())
for i in range(Config().getMAX_NUM()):
try:
resp = request.urlopen(req)
except urllib.error.URLError as e:
if i < Config().getMAX_NUM() - 1:
continue
else:
self.logger.error("{}:{}:次之后还是失败".format(
url,
Config().getMAX_NUM()))
return
content = resp.read()
soup = BeautifulSoup(content, "lxml")
return soup
def __init__(self):
super().__init__()
self.qifu = StartServer()
self.config = Config()
self.worldId_gong = int(self.config.get_conf("worldId_gong"))
self.worldId_pu1 = int(self.config.get_conf("worldId_pu1"))
self.worldId_pu2 = int(self.config.get_conf("worldId_pu2"))
self.areaId = int(self.config.get_conf("areaId"))
self.versionId = int(self.config.get_conf("versionId"))
self.groupname = time.strftime("%m.%d %H:%M:%S")
self.time = (datetime.datetime.utcnow() +
timedelta(hours=0.005)).isoformat() + 'Z'
def getData(self, url):
soup = SpiderUtil().getSoup(url)
data = {}
data['detail_url'] = url
chname = soup.find('div', class_="page-header").getText()
data["chname"] = re.sub("(\t|\n|\r|\040)*", "",chname)
results = soup.findAll('div', class_="panel panel-success")
for result in results:
tag = []
for content in result.contents:
if type(content) == bs4.Tag:
tag.append(content)
if len(tag) == 2:
if tag[0].getText() == "漏洞参数":
for p in tag[1].contents:
if type(p) == bs4.Tag:
text = p.getText().split(":")
if len(text) > 0 and text[0] in Config().getIVDVulList():
data[Config().getIVDVulList()[text[0]]] = text[1]
elif tag[0].getText() in Config().getIVDVulList():
data[Config().getIVDVulList()[tag[0].getText()]] = re.sub("(\t|\n|\r|\040)*", "", tag[1].getText())
self.ivdDao.insert(data)
def getUrlsByGoogle(self, keyWord):
datas = []
# 对url中中文处理
start_url = "https://www.google.com.hk/search?q=" + urllib.parse.quote(
keyWord)
browser = webdriver.Chrome()
browser.get(start_url)
while (self.isElementExist(browser, '下一页')):
browser.find_element_by_link_text('下一页').click()
soup = BeautifulSoup(browser.page_source,
"html.parser",
from_encoding='UTF-8')
self.getDataByGoogle(soup, keyWord, datas)
time.sleep(Config().getSleepTime())
browser.close()
self.urlKeyWordDao.insert(datas)
def load_logging_config():
current_dir = os.path.dirname(os.path.realpath(__file__))
log_config_path = os.path.join(current_dir, 'logging.yaml')
logs_dir = Config().get('logs_dir')
try:
with open(log_config_path, 'rt') as f:
timestamp = datetime.now().strftime("%d.%m.%Y.%H.%M.%S")
filepath = os.path.join(logs_dir, f'{timestamp}.log')
config = yaml.safe_load(f.read())
config['handlers']['empty_file_handler']['filename'] = filepath
config['handlers']['pretty_file_handler']['filename'] = filepath
except yaml.YAMLError as e:
raise Exception(f"Failed to parse '{log_config_path}'") from e
logging.config.dictConfig(config)
def xx(self, dataList):
dataCount = {}
pattern = re.compile(Config().getFilterFile2())
for data in dataList:
datas = data.url.split("/")
for key in datas:
match = pattern.match(key)
if match == None and key != '':
if not dataCount.get(key):
dataCount[key] = 1
else:
dataCount[key] += 1
sort = sorted(dataCount.items(), key=lambda e: e[1], reverse=True)
for item in sort:
print(item)
self.logger.debug(item)
def dataPreparation(self):
dataCount = {}
dataList = self.urlKeyWordDao.listURL()
pattern = re.compile(Config().getFilterFile())
for data in dataList:
datas = data.url.split(".")
for key in datas:
match = pattern.match(key)
if match == None and key != '':
if not dataCount.get(key):
dataCount[key] = 1
else:
dataCount[key] += 1
sort = sorted(dataCount.items(), key=lambda e: e[1], reverse=True)
for item in sort:
print(item)
self.logger.debug(item)
def __init__(self, taskId):
self.config = Config()
self.url = self.config.get_conf('url')
self.taskId = taskId
self.ins = ''
def __init__(self):
self.header = Config().getHeader()
self.logger = MyLog().getLogger()
self.cnnvdDao = CnnvdDao()
def __init__(self):
self.header = Config().getHeader()
self.logger = MyLog().getLogger()
self.gongKongDao = GongKongDao()
def __init__(self):
self.header = Config().getHeader()
self.logger = MyLog().getLogger()
self.urlKeyWordDao = UrlKeyWordDao()
def __init__(self):
self.config = Config()
self.url = self.config.get_conf('url') + "api/task/open-server"
class BasePage:
"""Base class to initialize the base page that will be called from all pages"""
url = None
_page_identifier_element = None
_page_load_timeout = Config().get('page_load_timeout')
_element_wait_timeout = Config().get('element_wait_timeout')
log = logging.getLogger('pretty_logger')
def __init__(self, driver):
self.driver = driver
self.verify_page_is_loaded()
# add logging to all methods
# def __getattribute__(self, name):
# returned = object.__getattribute__(self, name)
# if inspect.isfunction(returned) or inspect.ismethod(returned):
# self.log.info(f"{self.__class__.__name__}: called {returned.__name__}")
# return returned
def verify_page_is_loaded(self):
try:
self._is_page_identifier_element_displayed()
self._is_document_in_ready_state()
except TimeoutException:
message = f"{self.__class__.__name__} was not loaded."
self.log.exception(message)
# TODO this exception is not captured in logs
raise PageNotLoadedException(message)
def _is_page_identifier_element_displayed(self):
"""Wait until self._page_identifier element is displayed"""
try:
self.wait_until(EC.visibility_of_element_located(
self._page_identifier_element),
timeout=self._element_wait_timeout)
self.log.info(
f"{self.__class__.__name__}: Page identifier {self._page_identifier_element} was displayed."
)
except TimeoutException:
raise TimeoutException(
f"'{self.__class__.__name__}': Page identifier element {self._page_identifier_element} was not visible during {self._element_wait_timeout} seconds."
)
def _is_document_in_ready_state(self):
self.wait_until(document_has_ready_state())
self.log.info(
f"{self.__class__.__name__}: Document was in ready state.")
def is_loaded(self):
# TODO raises NoSuchElementException when cannot find element, should I handle and return False for not displayed - that may hide the fact that element is not present
return self.driver.find_element(
*self._page_identifier_element).is_displayed()
def is_not_loaded(self):
return not self.driver.find_element(
*self._page_identifier_element).is_displayed()
def get_url(self):
return self.driver.current_url
# region Wait methods
def wait_until(self, condition_method, timeout=5, **kwargs):
"""wait for the event in condition_method, log TimeoutException and re-raise"""
wait = WebDriverWait(self.driver, timeout, **kwargs)
try:
return wait.until(condition_method)
except TimeoutException:
raise TimeoutException(
f"Event '{condition_method.__class__.__name__}' did not occur during '{timeout}' seconds. Vars: {vars(condition_method)}"
)
def wait_until_displayed(self, locator, timeout=5, **kwargs):
try:
return self.wait_until(EC.visibility_of_element_located(locator),
timeout, **kwargs)
except TimeoutException:
raise ElementNotDisplayedException(
f"{self.__class__.__name__} page: Element {locator} was not displayed during {timeout} seconds."
)
def wait_until_clickable(self, locator, timeout=5, **kwargs):
try:
return self.wait_until(EC.element_to_be_clickable(locator),
timeout, **kwargs)
except TimeoutException:
raise ElementNotClickableException(
f"{self.__class__.__name__} page: Element {locator} was not clickable during {timeout} seconds."
)
# endregion
def _is_element_displayed(self, locator):
try:
return self.driver.find_element(*locator).is_displayed()
except NoSuchElementException:
return False
# region driver public API
def get_element(self, locator):
try:
element = self.driver.find_element(*locator)
self.log.info(
f"{self.__class__.__name__}: Element with locator {locator} was found."
)
return element
except NoSuchElementException:
self.log.exception(
f"{self.__class__.__name__}: Element with locator {locator} was not found."
)
raise
def enter_text(self, locator, text):
self.get_element(locator).send_keys(text)
self.log.info(
f"{self.__class__.__name__}: Entered text '{text}' into element with locator {locator}."
)
def click_element(self, locator):
self.get_element(locator).click()
self.log.info(
f"{self.__class__.__name__}: Clicked on element with locator {locator}."
)
def click_element_with_JS(self, locator):
element = self.get_element(locator)
self.driver.execute_script("arguments[0].click();", element)
self.log.info(
f"{self.__class__.__name__}: Clicked with JS on element with locator {locator}."
)
def get_text(self, locator):
element_text = self.get_element(locator).text
self.log.info(
f"{self.__class__.__name__}: Element with locator {locator} has text '{element_text}'."
)
return element_text
def get_element_attribute(self, locator, attribute):
element_attr = self.get_element(locator).get_attribute(attribute)
self.log.info(
f"{self.__class__.__name__}: Element with locator {locator} has attribute '{attribute}' with value '{element_attr}'."
)
return element_attr
def get_element_value(self, locator):
element_value = self.get_element(locator).get_attribute('value')
self.log.info(
f"{self.__class__.__name__}: Element with locator {locator} has value '{element_value}'."
)
return element_value
def switch_to_new_tab(self):
new_opened_tab_handle = self.driver.window_handles[1]
self.driver.switch_to.window(new_opened_tab_handle)
self.log.info(
f"{self.__class__.__name__}: Switched to new tab with handle {new_opened_tab_handle}."
)
def select_dropdown_option(self, locator, option_name):
Select(self.get_element(locator)).select_by_visible_text(option_name)
self.log.info(
f"{self.__class__.__name__}: Selected drop-down option '{option_name}'."
)
def mouse_over(self, locator):
element = self.get_element(locator)
ActionChains(self.driver).move_to_element(element).perform()
self.log.info(
f"{self.__class__.__name__}: Hovered moused over element with locator '{locator}'."
)
def __init__(self, session):
self.session = session
self.config = Config()
self.game_id = self.config.game_id
self.region_id = self.config.region_id
from selenium.webdriver.chrome.options import Options as ChromeOptions
from selenium.webdriver.firefox.options import Options as FirefoxOptions
from common.logger.custom_logger import load_logging_config
from common.data.credentials import get_credentials
from common.config.config import Config
from models.user import User
from pages.login_page import LoginPage
from pages.header_page import Header
from datetime import datetime
from py.xml import html
from pages.menu.main_menu import MainMenu
RESULTS_DIR = Config().get('results_dir')
SCREENSHOTS_DIR = Config().get('screenshots_dir')
LOGS_DIR = Config().get('logs_dir')
@pytest.fixture(scope="session", autouse=True)
def create_results_dirs(config):
Path(RESULTS_DIR).mkdir(parents=True, exist_ok=True)
Path(SCREENSHOTS_DIR).mkdir(parents=True, exist_ok=True)
Path(LOGS_DIR).mkdir(parents=True, exist_ok=True)
load_logging_config()
empty_logger = logging.getLogger('empty_logger')
pretty_logger = logging.getLogger('pretty_logger')
def config():
return Config()