def _crawl(detail_link: str, driver: WebDriver) -> Info:
driver.get(detail_link)
time.sleep(5)
accordeon = driver.find_elements_by_class_name('accordeon')
try:
driver.find_element_by_class_name(
'moove-gdpr-infobar-allow-all').click(
) # get rid of cookie banner blocking all clicks on this website
except:
pass
info = _get_all_present_fields(accordeon, driver)
return _convert(info)
def _get_detail_links(driver: WebDriver) -> List[str]:
time.sleep(10) # website is very slow
detail_links = []
ngo_list = driver.find_element_by_id('toplist')
links = ngo_list.find_elements_by_tag_name('a')
for link in links:
link_href = link.get_attribute("href")
detail_links.append(link_href)
return detail_links
def _crawl_detail_page(driver: WebDriver) -> List[DetailPageInfo]:
detail_pages_infos = []
for idx in range(1, 315):
driver.get(URL)
rows: List[WebElement] = _get_table_rows(driver)
for row in rows:
cols = row.find_elements_by_tag_name('td')
row_idx: int = extract_idx(cols[0])
if row_idx == idx:
link_element = cols[0].find_element_by_tag_name('a')
link_element.click()
detail_page_info = _extract_detail_info(driver, idx)
detail_pages_infos.append(detail_page_info)
break
_clean_detail_pages(detail_pages_infos)
return detail_pages_infos
def ie(path='./IEDriverServer.exe') -> WebDriver:
"""
IE
:param path:IE Driver路径
:return: WebDriver
"""
dr = IE(executable_path=path)
dr.set_page_load_timeout(30)
dr.implicitly_wait(10)
dr.maximize_window()
return dr
def _extract_soft_facts(driver: WebDriver) -> SoftFacts:
elements = driver.find_elements_by_tag_name('div')
soft_facts_elements = elements[36]
soft_facts_list = soft_facts_elements.find_elements_by_tag_name('p')
aims = soft_facts_list[0].text.strip()
activities = soft_facts_list[1].text.strip()
accreditations = soft_facts_list[3].text.strip()
areas_of_competence = _split_into_areas(soft_facts_list[7].text.strip())
geographical_representation = _split_into_countries(
soft_facts_list[8].text.strip())
return SoftFacts(aims, activities, accreditations, areas_of_competence,
geographical_representation)
def _extract_hard_facts(driver: WebDriver, idx: int) -> HardFacts:
hard_fact_element = driver.find_elements_by_tag_name('div')[12]
website = hard_fact_element.find_elements_by_tag_name(
'span')[1].text.strip()
president_element = hard_fact_element.find_elements_by_class_name('row')[4]
president_name = president_element.find_element_by_tag_name(
'h4').text.strip()
founding_year, staff_number, members_number, languages = _extract_quick_facts(
driver, idx)
return HardFacts(website, president_name, founding_year, staff_number,
members_number, languages)
def getWebDriver(self, browser):
print(browser)
if browser.upper() == "IE":
driver = InternetExplorerDriver()
driver.maximize_window()
self.logger.appendContent("新建IE驱动")
return driver
elif browser.upper() == "CHROME":
options = ChromeOptions()
options.add_argument("test-type")
driver = ChromeDriver(chrome_options=options)
driver.maximize_window()
self.logger.appendContent("新建chrome驱动")
return driver
elif browser.upper() == "FIREFOX":
driver = FirefoxDriverDriver()
driver.maximize_window()
self.logger.appendContent("新建FireFox驱动")
return driver
else:
return None
def batchAccept(browser: WebDriver):
# 批量受理
u = 'http://10.204.14.35/eoms4/sheetBpp/myWaitingDealSheetQueryty.action?baseSchema=WF4_EL_TTM_TTH_EQU&var_pagesize=100'
browser.get(u)
total_ele = 'form#form1 span.pagenumber'
total_element = browser.find_element(By.CSS_SELECTOR, total_ele)
total = re.search(r'共([0-9]+)条数据', total_element.text).group(1)
if total < 1:
return
todo_list = browser.find_elements(By.CSS_SELECTOR, 'table#tab tr')[1:]
data = []
i = 0
for x in range(0, int(total)):
title = todo_list[2 * x + 1].text
if title.find('[数据网]') != -1:
checkbox = todo_list[2 * x].find_element(By.NAME, 'checkid')
checkbox.click()
# batchAccept_btn = browser.find_element(By.CSS_SELECTOR, 'li.page_active_button')
browser.execute_script('batchAccept();')
def jumpToEOMS(browser: WebDriver, keyword='') -> list:
eoms_url = 'http://uip.ln.cmcc/_layouts/Document/BridgeToSPControl.aspx?skipcode=emoss'
browser.get(eoms_url)
# 故障处理工单(设备)：
todo_list_url = 'http://10.204.14.35/eoms4/sheetBpp/myWaitingDealSheetQueryGlobalTemplate.action?baseSchema=WF4_EL_TTM_TTH_EQU&var_pagesize=100'
browser.get(todo_list_url)
total_ele = 'form#form1 span.pagenumber'
total_element = browser.find_element(By.CSS_SELECTOR, total_ele)
total = re.search(r'共([0-9]+)条数据', total_element.text).group(1)
# oo = $('table#tab tr')
# Array.prototype.shift.apply(oo)
todo_list = browser.find_elements(By.CSS_SELECTOR, 'table#tab tr')[1:]
def get_url(title_ele):
onclick = title_ele.find_element_by_tag_name('a').get_attribute(
'onclick')
args = re.findall(r'\'(.*?)\'', onclick)
# browser.execute_script('')
url = 'http://10.204.14.35/eoms4/sheet/openWaittingSheet.action?baseSchema={}&taskid={}&baseId={}&entryId=&version=&processType={}'
url = url.format(args[0], args[2], args[1], args[3])
# url = 'http://10.204.14.31:8001/bpp/ultrabpp/view.action?baseSchema={}&baseId={}&taskid={}&processType={}'
# url = url.format(args[0], args[1], args[2], args[3])
return url
data = []
# 待处理工单统计
i = 0
for x in range(0, int(total)):
title = todo_list[2 * x + 1]
if title.text.find(keyword) != -1:
data.append({})
data[i]['title'] = title.text[21:]
content = todo_list[2 * x].find_elements_by_tag_name('td')
data[i]['end_time'] = datep(content[4].text)
data[i]['find_time'] = datep(content[5].text)
data[i]['status'] = content[6].text
data[i]['url'] = get_url(title)
i += 1
# 上清除时间的工单统计
data2 = []
# 未上清除时间统计
data3 = []
j = 0
for x in range(i):
browser.get(data[x]['url'])
data[x]['clear_time'] = browser.find_element_by_id(
'INC_Alarm_ClearTime').get_attribute('value')
title_match = re.search(r'([A-Z\-0-9]+) 上报 (.+)', data[x]['title'])
try:
data[x]['device'] = title_match.group(1)
data[x]['event'] = title_match.group(2)
except:
pass
if data[x]['clear_time'] != '':
data2.append(data[x])
j += 1
else:
data3.append(data[x])
# TODO 详细推送
msg_title = '{}故障工单 {} 个，已上清除 {} 个。'
msg_title = msg_title.format(keyword, len(data), len(data2))
find_time2, find_time3 = [], []
if data2 != []:
for x in data2:
find_time2.append(datef(x['find_time']))
if data3 != []:
for x in data3:
find_time3.append(datef(x['find_time']))
msg_text = '### {}\n\n**已上清除建单时间分别为**：\n\n{}\n\n**未上清除建单时间分别为**：\n\n{}\n\n> 推送时间：{}'
t2, t3 = '\n\n'.join(find_time2), '\n\n'.join(find_time3)
msg_text = msg_text.format(msg_title, t2, t3, datef())
send_msg(msg_markdown(msg_title, msg_text, True))
browser.get(eoms_url)
return data, data2, data3
def __init__(self, *args, **kwargs):
IeWebDriver.__init__(self, *args, **kwargs)
WaitUntil.__init__(self)
def cleanUpCookiesAndLaunchIE(self):
caps = DesiredCapabilities.INTERNETEXPLORER
ie = WebDriver()
ie.delete_all_cookies()
ie.get("http://www.yahoo.com")
return ie
def cleanUpCookiesAndLaunchIE(self):
caps = DesiredCapabilities.INTERNETEXPLORER
ie = WebDriver()
ie.delete_all_cookies()
ie.get("http://www.yahoo.com")
return ie
def __init__(self):
""" Create IE Driver Wrapper"""
WebDriver.__init__(self)
DriverWrapper.__init__(self)
def _get_table_rows(driver: WebDriver) -> List[WebElement]:
time.sleep(1) # if we don't wait here, the table might not be loaded yet
table = driver.find_element_by_tag_name('table')
body = table.find_element_by_tag_name('tbody')
rows = body.find_elements_by_tag_name('tr')
return rows
def _extract_quick_facts(
driver: WebDriver, idx: int
) -> Tuple[Optional[int], Optional[int], Optional[int], List[str]]:
founding_year = None
staff_number = None
members_number = None
languages: List[str] = []
possible_quick_facts_elements = driver.find_elements_by_class_name(
'ng-scope')
likely_quick_facts_elements = [
p for p in possible_quick_facts_elements
if p.find_elements_by_tag_name('strong')
]
founded = [
l for l in likely_quick_facts_elements if l.text.startswith('Founded')
]
staff = [
l for l in likely_quick_facts_elements
if l.text.strip().endswith('staff')
]
members = [
l for l in likely_quick_facts_elements
if l.text.strip().endswith('members')
]
working_languages = [
l for l in likely_quick_facts_elements
if l.text.startswith('Working languages')
]
if founded:
founding_year = int(
founded[0].find_element_by_tag_name('strong').text.strip())
if staff:
staff_string = staff[0].find_element_by_tag_name('strong').text.strip()
staff_string = staff_string.replace(' ', '')
try:
staff_number = int(staff_string)
except:
if staff_string == '6fulltime+2interns':
staff_number = 6
if staff_string == '2.8':
staff_number = 3
if staff_string == '3employeesand7volunteers':
staff_number = 3
if staff_string == '5employees':
staff_number = 5
print(f'SKIPPING STAFF ({idx}) - {staff_string}')
if members:
members_string = members[0].find_element_by_tag_name(
'strong').text.strip()
members_string = members_string.replace(' ', '')
members_string = members_string.replace('+', '')
try:
members_number = int(members_string)
except:
if members_string == '384organisations':
members_number = 384
if members_string == '50delegations':
members_number = 50
if members_string == '120,000':
members_number = 120000
if members_string == 'Around35':
members_number = 35
if members_string == 'about5000':
members_number = 5000
if members_string == '46nationaluniversitysportsgoverningbodies':
members_number = 46
if members_string == 'around1000membersin34branches':
members_number = 1000
if members_string == '44associations':
members_number = 44
if members_string == 'approx.700':
members_number = 700
if members_string == '5,000,000':
members_number = 5000000
if members_string == '250associations':
members_number = 250
if members_string == '28memberorganisations':
members_number = 28
if working_languages:
languages = working_languages[0].find_element_by_tag_name(
'strong').text.strip()
return founding_year, staff_number, members_number, languages