def handle_room_tag(driver: webdriver, comm: str):
"""
进入每一个直播, 并插播广告
写入日志
"""
# 将所有标签统计出来
driver.execute_script(Order.page_end.value)
sum_room = driver.find_elements_by_class_name(Order.room_tag.value)
i = 0
while i < len(sum_room):
try:
_into_room_handle(driver, comm, i)
except ElementClickInterceptedException:
move_down(driver)
_into_room_handle(driver, comm, i)
i += 1
try:
move_down(driver)
tag = driver.find_element_by_css_selector(Order.page_down.value)
if tag:
time.sleep(1)
tag.click()
driver.implicitly_wait(5)
handle_room_tag(driver, comm)
except NoSuchElementException:
print("finish")
driver.close()
return
def test_invalid_custom_docker(driver: selenium.webdriver, *args, **kwargs):
"""
Test invalid custom Docker instructions.
Args:
driver
"""
r = testutils.prep_py3_minimal_base(driver)
username, project_name = r.username, r.project_name
# add an invalid custom docker instruction
logging.info("Adding invalid custom Docker")
environment = testutils.EnvironmentElements(driver)
environment.environment_tab_button.click()
time.sleep(1)
driver.execute_script("window.scrollBy(0, 600);")
environment.custom_docker_edit_button.click()
time.sleep(1)
environment.custom_docker_text_input.send_keys("RUN /bin/false")
time.sleep(1)
driver.execute_script("window.scrollBy(0, 300);")
environment.custom_docker_save_button.click()
# wait until container status is stopped
wait = selenium.webdriver.support.ui.WebDriverWait(driver, 200)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".flex>.Rebuild")))
time.sleep(2)
# assert container status is stopped and 'Successfully tagged' is in footer
envelts = testutils.elements.EnvironmentElements(driver)
assert driver.find_element_by_css_selector(".flex>.Rebuild").is_displayed(), "Expected rebuild container status"
assert "Project failed to build" in driver.find_element_by_css_selector(".Footer__message-title").text, \
"Expected 'Project failed to build' in footer"
def execSearch(browser: webdriver, URL):
"""
Googleで検索を実行する
:param browser: webdriver
"""
# スクリーンショットのファイル名用に日付を取得
dt = datetime.datetime.today()
dtstr = dt.strftime("%Y%m%d%H%M%S")
# Googleにアクセス
# スクショしたい画面を表示
browser.get(URL)
sleep(1)
# # キーワードの入力
# search_box = browser.find_element_by_name("q")
# search_box.send_keys('docker selenium')
# # 検索実行
# search_box.submit()
sleep(1)
# 画面全体をスクショするための調整
w = browser.execute_script('return document.body.scrollWidth')
h = browser.execute_script('return document.body.scrollHeight')
browser.set_window_size(w, h)
# スクリーンショット
browser.save_screenshot('images/' + dtstr + '.png')
def scroll_to_xy(driver: webdriver, x, y):
'''
用途：通过制定xy坐标来滑动web页面
:param driver: webdriver
:param x:屏幕向右移动的距离
:param y:屏幕向下移动的距离
:return:
1、滚动到文档中的某个坐标
window.scrollTo(x-coord,y-coord )
window.scrollTo(options)
·x-coord 是文档中的横轴坐标。
·y-coord 是文档中的纵轴坐标。
·options 是一个包含三个属性的对象:
·top 等同于 y-coord
·left 等同于 x-coord
·behavior 类型String,表示滚动行为,支持参数 smooth(平滑滚动),instant(瞬间滚动),默认值auto,实测效果等同于instant
例子：
window.scrollTo( 0, 1000 );
// 设置滚动行为改为平滑的滚动
window.scrollTo({
top: 1000,
behavior: "smooth"
});
'''
js = """
window.scrollTo("{}", "{}")
""".format(x, y)
driver.execute_script(js)
def test_valid_custom_docker(driver: selenium.webdriver, *args, **kwargs):
"""
Test valid custom Docker instructions.
Args:
driver
"""
r = testutils.prep_py3_minimal_base(driver)
username, project_name = r.username, r.project_name
wait = selenium.webdriver.support.ui.WebDriverWait(driver, 200)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".flex>.Stopped")))
logging.info("Adding valid custom Docker")
environment = testutils.EnvironmentElements(driver)
environment.environment_tab_button.click()
driver.execute_script("window.scrollBy(0, 600);")
environment.custom_docker_edit_button.click()
environment.custom_docker_text_input.send_keys("RUN cd /tmp && "
"git clone https://github.com/gigantum/confhttpproxy && "
"cd /tmp/confhttpproxy && pip install -e.")
time.sleep(1)
driver.execute_script("window.scrollBy(0, 300);")
environment.custom_docker_save_button.click()
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".flex>.Stopped")))
# assert container status is stopped and 'Successfully tagged' is in footer
time.sleep(15)
assert driver.find_element_by_css_selector(".flex>.Stopped").is_displayed(), "Expected stopped container"
def fetch_image_urls(query: str,
max_links_to_fetch: int,
wd: webdriver,
sleep_between_interactions: int = 1):
def scroll_to_end(wd):
wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(sleep_between_interactions)
# build the google query
search_url = "https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q={q}&oq={q}&gs_l=img"
# load the page
wd.get(search_url.format(q=query))
image_urls = set()
image_count = 0
results_start = 0
while image_count < max_links_to_fetch:
scroll_to_end(wd)
# get all image thumbnail results
thumbnail_results = wd.find_elements_by_css_selector("img.rg_ic")
number_results = len(thumbnail_results)
print(
f"Found: {number_results} search results. Extracting links from {results_start}:{number_results}"
)
for img in thumbnail_results[results_start:number_results]:
# try to click every thumbnail such that we can get the real image behind it
try:
img.click()
time.sleep(sleep_between_interactions)
except Exception:
continue
# extract image urls
actual_images = wd.find_elements_by_css_selector('img.irc_mi')
for actual_image in actual_images:
if actual_image.get_attribute('src'):
image_urls.add(actual_image.get_attribute('src'))
image_count = len(image_urls)
if len(image_urls) >= max_links_to_fetch:
print(f"Found: {len(image_urls)} image links, done!")
break
else:
print("Found:", len(image_urls),
"image links, looking for more ...")
time.sleep(1)
load_more_button = wd.find_element_by_css_selector(".ksb")
if load_more_button:
wd.execute_script("document.querySelector('.ksb').click();")
# move the result startpoint further down
results_start = len(thumbnail_results)
return image_urls
def fetch_image_urls(query: str,
max_links_to_fetch: int,
wd: webdriver,
sleep_between_interactions: int = 1):
#search for query
wd.get('https://www.google.ae/imghp?hl=en&ogbl')
search_box = wd.find_element_by_css_selector('input.gLFyf')
search_box.send_keys(query + ' company logo')
search_box.submit()
image_urls = set()
image_count = 0
results_start = 0
while image_count < max_links_to_fetch:
# get all image thumbnail results
thumbnail_results = wd.find_elements_by_css_selector("img.Q4LuWd")
number_results = len(thumbnail_results)
print(
f"Found: {number_results} search results. Extracting links from {results_start}:{number_results}"
)
for img in thumbnail_results[results_start:number_results]:
# try to click every thumbnail such that we can get the real image behind it
try:
time.sleep(sleep_between_interactions)
img.click()
time.sleep(sleep_between_interactions)
except Exception:
continue
# extract image urls
actual_images = wd.find_elements_by_css_selector('img.n3VNCb')
for actual_image in actual_images:
if actual_image.get_attribute(
'src') and 'http' in actual_image.get_attribute('src'):
image_urls.add(actual_image.get_attribute('src'))
image_count = len(image_urls)
if len(image_urls) >= max_links_to_fetch:
print(f"Found: {len(image_urls)} image links, done!")
break
else:
print("Found:", len(image_urls),
"image links, looking for more ...")
time.sleep(30)
return
load_more_button = wd.find_element_by_css_selector(".mye4qd")
if load_more_button:
wd.execute_script("document.querySelector('.mye4qd').click();")
# move the result startpoint further down
results_start = len(thumbnail_results)
return image_urls
def mozilla_input_page(browser: webdriver) -> MozillaInputPage:
mozilla_input_page = MozillaInputPage(driver=browser)
mozilla_input_page.open_page()
frame = mozilla_input_page.find_element(
locator=mozilla_input_page.FRAME_BASIC_EXAMPLE)
browser.execute_script("arguments[0].scrollIntoView();", frame)
browser.switch_to.frame(frame_reference=frame)
return mozilla_input_page
def scrollToEnd(wd: webdriver, loops: int = 1, interval: int = 1):
try:
for _ in range(loops):
wd.execute_script(
'window.scrollTo(0, document.body.scrollHeight);')
sleep(interval)
except JavascriptException:
pass
def fetch_image_urls(query: str, max_links_to_fetch: int, wd: webdriver, sleep_between_interactions: int = 1):
def scroll_to_end(wd):
wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(sleep_between_interactions)
# build the google query
search_url = "https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q={q}&oq={q}&gs_l=img" # manual google image url
# {q} - search string we wanted to searvj
# https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q=dog&oq=dog&gs_l=img
# load the page - this give you the dog images
wd.get(search_url.format(q=query)) # this function opens up the browser with the searched string
image_urls = set() # we want the unique url and not the duplicate ones ie why set is used.
image_count = 0
results_start = 0
while image_count < max_links_to_fetch:
scroll_to_end(wd) # makes you browser scroll down
# get all image thumbnail results
thumbnail_results = wd.find_elements_by_css_selector("img.Q4LuWd") # binary format of the results
number_results = len(thumbnail_results) # length of the results
print(f"Found: {number_results} search results. Extracting links from {results_start}:{number_results}")
for img in thumbnail_results[results_start:number_results]:
# try to click every thumbnail such that we can get the real image behind it
try:
img.click() # clicks the image
time.sleep(sleep_between_interactions) # wait for some time so image will be loaded
except Exception:
continue
# extract image urls
actual_images = wd.find_elements_by_css_selector('img.n3VNCb') # after clicking fetch the image
for actual_image in actual_images:
if actual_image.get_attribute('src') and 'http' in actual_image.get_attribute('src'): # if it is holding a valid url then add
image_urls.add(actual_image.get_attribute('src'))
image_count = len(image_urls)
if len(image_urls) >= max_links_to_fetch:
print(f"Found: {len(image_urls)} image links, done!")
break
else:
print("Found:", len(image_urls), "image links, looking for more ...")
time.sleep(30)
return
load_more_button = wd.find_element_by_css_selector(".mye4qd")
if load_more_button:
wd.execute_script("document.querySelector('.mye4qd').click();")
# move the result startpoint further down
results_start = len(thumbnail_results)
return image_urls
def scroll_to_element(driver: webdriver, element):
"""
用途：滚动屏幕使得元素上下、左右居中
:param driver:
:param element:
:return:
"""
js = "arguments[0].scrollIntoView({behavior: 'smooth', block: 'center', inline: 'center'});"
driver.execute_script(js, element)
def fetch_image_urls(query: str,
max_links_to_fetch: int,
wd: webdriver,
sleep_between_interactions: int = 1):
def scroll_to_end(wd):
wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(sleep_between_interactions)
search_url = "https://www.google.com/search?q={q}&source=lnms&tbm=isch"
# load the page
wd.get(search_url.format(q=query))
image_urls = set()
image_count = 0
results_start = 0
while image_count < max_links_to_fetch:
for _ in range(10):
scroll_to_end(wd)
# get all image thumbnail results
thumbnail_results = wd.find_elements_by_css_selector("img.Q4LuWd")
number_results = len(thumbnail_results)
for img in thumbnail_results[results_start:number_results]:
if img.get_attribute('src') and 'http' in img.get_attribute('src'):
image_urls.add(img.get_attribute('src'))
if img.get_attribute('src') and 'data' in img.get_attribute('src'):
image_urls.add(img.get_attribute('src'))
image_count = len(image_urls)
if len(image_urls) >= max_links_to_fetch:
print(f"Found: {len(image_urls)} image links, done!")
break
else:
print("Found:", len(image_urls),
"image links, looking for more ...")
# return
load_more_button = wd.find_element_by_css_selector(".mye4qd")
if load_more_button:
wd.execute_script("document.querySelector('.mye4qd').click();")
time.sleep(3)
# end_of_page = wd.find_element_by_xpath("//div[@class='OuJzKb Yu2Dnd']")
end_of_page = wd.find_elements_by_xpath(
"//*[ contains (text(), 'Looks like') ]")
if end_of_page:
print("end of the page")
break
# move the result startpoint further down
results_start = len(thumbnail_results)
return image_urls
def fetch_image_urls(query: str,
max_links_to_fetch: int,
wd: webdriver,
sleep_between_interactions: int = 1):
def scroll_to_end(wd):
wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(sleep_between_interactions)
search_url = "https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q={q}&oq={q}&gs_l=img"
wd.get(search_url.format(q=query))
image_urls = set()
image_count = 0
results_start = 0
while image_count < max_links_to_fetch:
scroll_to_end(wd)
thumbnail_results = wd.find_elements_by_css_selector("img.Q4LuWd")
number_results = len(thumbnail_results)
print(
f"Found: {number_results} search results. Extracting links from {results_start}:{number_results}"
)
for img in thumbnail_results[results_start:number_results]:
try:
img.click()
time.sleep(sleep_between_interactions)
except Exception:
continue
actual_images = wd.find_elements_by_css_selector('img.n3VNCb')
for actual_image in actual_images:
if actual_image.get_attribute(
'src') and 'http' in actual_image.get_attribute('src'):
image_urls.add(actual_image.get_attribute('src'))
image_count = len(image_urls)
if len(image_urls) >= max_links_to_fetch:
print(f"Found: {len(image_urls)} image links, done!")
break
else:
print("Found:", len(image_urls),
"image links, looking for more ...")
time.sleep(30)
return
load_more_button = wd.find_element_by_css_selector(".mye4qd")
if load_more_button:
wd.execute_script("document.querySelector('.mye4qd').click();")
results_start = len(thumbnail_results)
return image_urls
def remarcar_elemento(navegador: webdriver, xpath, estilo='2px solid red'):
# Buscamos el elemento
elemento = navegador.find_element_by_xpath(xpath)
# Recuperamos el borde que tenia
original = navegador.execute_script("return arguments[0].style.border;",
elemento)
# Ponemos el nuevo borde
navegador.execute_script("arguments[0].style.border = '" + estilo + "';",
elemento)
return original
def fetch_image_urls(query:str,max_links_to_fetch:int,wd:webdriver,sleep_between_interaction: int =1):
def scroll_to_end(wd):
wd.execute_script("window.scrollTo(0,document.body.scrollHeight);")
time.sleep(sleep_between_interaction)
search_url = "https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q={q}&oq={q}&gs_l=img"
#load the page
wd.get(search_url.format(q=query))
image_urls=set()
image_count= 0
result_start=0
while image_count <max_links_to_fetch:
scroll_to_end(wd)
#get all the images thumbnails results
thumbnails_result= wd.find_elements_by_css_selector("img.Q4LuWd")
#print(thumbnails_result)
num_results= len(thumbnails_result)
print(f"Found: {num_results} search results. Extracting links from {result_start} :{num_results}")
for img in thumbnails_result[result_start:num_results]:
#try to click every thumbnail and get the image page behind it.
try:
img.click()
time.sleep(sleep_between_interaction)
except Exception:
continue
#extract images urls
actual_images=wd.find_elements_by_css_selector('img.n3VNCb')
for actual_image in actual_images:
if actual_image.get_attribute('src') and 'http' in actual_image.get_attribute('src'):
image_urls.add(actual_image.get_attribute('src'))
image_count=len(image_urls)
if len(image_urls) >= max_links_to_fetch:
print(f"Found: {len(image_urls)} image links ,Done!!")
break
else:
print(f"Found {len(image_urls)} images links ,Looking for more")
time.sleep(30)
return
load_more_button=wd.find_element_by_css_selector('.mye4qd')
if load_more_button:
wd.execute_script("document.querySelector('.mye4qd').click();")
#move the result startpoint further down
result_start=len(thumbnails_result)
return image_urls
def scroll_to_element(brow: webdriver,
element: webdriver,
position='{block: "center"}') -> None:
"""
If the argument of 'scrollIntoView' is 'true' the command scrolls
the webpage positioning the element at the top of the window, if it
is 'false' the element will be positioned at the bottom.
"""
script = f'return arguments[0].scrollIntoView({position});'
brow.execute_script(script, element)
def fetch_image_urls(search_term: str, n_links: int, web_driver: webdriver, sleep_between_interactions: int = 1):
def scroll_to_end(web_driver):
web_driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
time.sleep(sleep_between_interactions)
search_url = f"https://www.google.com/search?tbm=isch&q={'+'.join(search_term.split())}"
web_driver.get(search_url)
image_counter = 0
res_start = 0
image_urls = set()
while(image_counter < n_links):
thumbnail_results = web_driver.find_elements_by_css_selector(
'img.Q4LuWd')
number_results = len(thumbnail_results)
scroll_to_end(web_driver)
print(
f'Found {number_results} search results. Extracting links {res_start} to {number_results}')
for img in thumbnail_results[res_start:number_results]:
try:
img.click()
time.sleep(sleep_between_interactions)
except:
continue
actual_image = web_driver.find_elements_by_css_selector(
'img.n3VNCb')
for img in actual_image:
src = img.get_attribute('src')
if src and 'http' in src:
image_urls.add(src)
image_counter = len(image_urls)
if image_counter >= n_links:
print(f"Found {n_links} image links.")
break
else:
print("Found:", len(image_urls),
"image links, looking for more ...")
time.sleep(3)
load_more_button = web_driver.find_element_by_css_selector(
".mye4qd")
if load_more_button:
web_driver.execute_script(
"document.querySelector('.mye4qd').click();")
res_start = len(thumbnail_results)
return image_urls
def fetch_image_urls(query: str, max_links_to_fetch: int, wd: webdriver, sleep_between_interactions: float = 1.0):
def scroll_to_end(wd):
wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(sleep_between_interactions)
search_url = "https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q={q}&oq={q}&gs_l=img"
# load the page
wd.get(search_url.format(q=query))
image_urls = set()
image_count = 0
results_start = 0
while image_count < max_links_to_fetch:
scroll_to_end(wd)
# get all image thumbnail results
thumbnail_results = wd.find_elements_by_css_selector("img.Q4LuWd")
number_results = len(thumbnail_results)
logger.info("Found: %i search results. Extracting links from %i:%i" % (number_results, results_start, number_results))
for img in thumbnail_results[results_start:number_results]:
# try to click every thumbnail such that we can get the real image behind it
try:
img.click()
time.sleep(sleep_between_interactions)
except Exception:
continue
# extract image urls
actual_images = wd.find_elements_by_css_selector("img.n3VNCb")
for actual_image in actual_images:
if actual_image.get_attribute("src") and "http" in actual_image.get_attribute("src"):
image_urls.add(actual_image.get_attribute("src"))
image_count = len(image_urls)
if len(image_urls) >= max_links_to_fetch:
logger.info("Found: %i image links, done!" % len(image_urls))
break
else:
logger.info("Found: %i image links, looking for more ..." % len(image_urls))
time.sleep(30)
load_more_button = wd.find_element_by_css_selector(".mye4qd")
if load_more_button:
wd.execute_script("document.querySelector('.mye4qd').click();")
# move the result startpoint further down
results_start = len(thumbnail_results)
return image_urls
def fetch_image_urls(query: str, max_links_to_fetch: int, wd: webdriver, sleep_between_interactions: int = 1):
"""Allows the webdriver to look for a query in Google Image and fetches a number of image links
corresponding to the query."""
# build the google query
search_url = "https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q={q}&oq={q}&gs_l=img"
# load the page
wd.get(search_url.format(q=query))
image_urls = set()
image_count = 0
results_start = 0
while image_count < max_links_to_fetch:
_scroll_to_end(wd)
# get all image thumbnail results
thumbnail_results = wd.find_elements_by_css_selector("img.Q4LuWd")
number_results = len(thumbnail_results)
print(f"Found: {number_results} search results. Extracting links from {results_start}:{number_results}")
for img in thumbnail_results[results_start:number_results]:
# try to click every thumbnail such that we can get the real image behind it
try:
img.click()
time.sleep(sleep_between_interactions)
except Exception:
continue
# extract image urls
actual_images = wd.find_elements_by_css_selector('img.n3VNCb')
for actual_image in actual_images:
if actual_image.get_attribute('src') and 'http' in actual_image.get_attribute('src'):
if actual_image.get_attribute('src') is not None:
image_urls.add(actual_image.get_attribute('src'))
else:
pass
image_count = len(image_urls)
if len(image_urls) >= max_links_to_fetch:
print(f"Found: {len(image_urls)} image links, done!")
break
else:
print("Found:", len(image_urls), "image links, looking for more ...")
time.sleep(30)
load_more_button = wd.find_element_by_css_selector(".mye4qd")
if load_more_button:
wd.execute_script("document.querySelector('.mye4qd').click();")
# move the result start point further down
results_start = len(thumbnail_results)
return image_urls
def height_lig(driver: webdriver, element):
"""
用途：指定元素高亮
:param driver: webdriver
:param element: 需要高亮的元素
:return:
"""
js = """
arguments[0].style.border="2px solid red";
"""
driver.execute_script(js, element)
def remove_attr(element, attribute, driver: webdriver):
"""
用途：以下方法可以删除元素的任何属性
:param element: 需要被删除属性的控件
:param attribute: 需要删除的属性
:param driver: webdriver
:return:
"""
js = """
arguments[0].removeAttribute("{attr}");
""".format(attr=attribute)
driver.execute_script(js, element)
def __paginate(driver: webdriver, pagination_index: int, timeout=2) -> object:
"""
Attempt to paginate
:param driver: webdriver
:param pagination_index: int
:return: object
"""
elapsed = 0
sleep_time = 0.05
clicked = False
pagination_btn_selector = Selectors.paginationBtn(pagination_index)
# Wait pagination html to load
try:
WebDriverWait(driver, 5).until(ec.presence_of_element_located((By.CSS_SELECTOR, Selectors.pagination)))
except:
return {'success': False, 'error': 'Timeout on pagination'}
# Try click next pagination button (if exists)
clicked = driver.execute_script(
'''
const btn = document.querySelector(arguments[0]);
if (btn) {
btn.click();
return true;
}
else {
return false;
}
''',
pagination_btn_selector)
# Failed to click next pagination button (pagination exhausted)
if not clicked:
return {'success': False, 'error': 'Pagination exhausted'}
# Wait for new jobs to load
while elapsed < timeout:
loaded = driver.execute_script(
'''
return document.querySelectorAll(arguments[0]).length > 0;
''',
Selectors.links)
if loaded:
return {'success': True}
sleep(sleep_time)
elapsed += sleep_time
return {'success': False, 'error': 'Timeout on pagination'}
def height_light(driver: webdriver, css, index=0):
"""
用途：方便用户查看当前操作的是哪个页面元素，也方便测试人员定位问题
:param driver: webdriver
:param css: css选择器
:param index: 列表的下标
:return:
"""
js = """
var element = document.querySelectorAll("{css}")[{index}];
element.style.border="4px solid red";
""".format(css=css, index=index)
driver.execute_script(js)
def window_scroll(driver: webdriver, element, x, y):
"""
用途：指定元素移动的某位置
:param driver: webdriver
:param element: 指定元素
:param x:屏幕向右移动的距离
:param y:屏幕向下移动的距离
:return:
"""
js = """
arguments[0].scrollTo("{}", "{}")
""".format(x, y)
driver.execute_script(js, element)
def click(driver: webdriver, css, index=None, describe=None):
"""
用途：由于web自动化的最大问题就是稳定性比较差，有些时候使用selenium无法点击元素，因此我们可以使用JS实现元素的点击操作
:param self:
:param driver: webdriver
:param css: css选择器
:param index: 列表的下标
:param describe: 描述信息
:return:
"""
js = """var elm = document.querySelectorAll("{css}")[{index}];
elm.style.border="2px solid red";
elm.click();""".format(css=css, index=_set_index(index))
driver.execute_script(js)
def clear(driver: webdriver, css,index=None, describe=None):
"""
用途：用来清除输入框的内容
:param driver: webdriver
:param css: css选择器
:param index: 列表的下标
:param describe: 描述信息
:return:
"""
js = """var elm = document.querySelectorAll("{css}")[{index}];
elm.style.border="2px solid red";
elm.value = "";""".format(css=css, index=_set_index(index))
driver.execute_script(js)
def fetch_image_urls(search_keyword: str, download_number: int, wd: webdriver):
print('------------------------------------------------')
print('Start getting thumnails')
fetch_thumbnail_count = 0
thumbnails = None
# fetch thumbnails up to number of images to download
while fetch_thumbnail_count < download_number:
wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(1)
thumbnails = wd.find_elements_by_css_selector('img.Q4LuWd')
fetch_thumbnail_count = len(thumbnails)
# break if fetched thumbnails number exceed number of images to download
if len(thumbnails) >= download_number:
print('Success: Fetched thumbnails count', download_number)
break
else:
# load more thumbnails when load_more_button appears
load_more_button = wd.find_element_by_css_selector(".mye4qd")
if load_more_button:
wd.execute_script("document.querySelector('.mye4qd').click();")
# brek when end_text appears ( this is the limit of thumbnails that can be fetched )
end_text = wd.find_element_by_class_name('OuJzKb')
if end_text and end_text.text == 'Looks like you\'ve reached the end':
print('Success: Fetched maximum thumbnails count', len(thumbnails))
break
print('Start getting image urls')
image_urls = []
# extract the image url from the elements displayed by clicking the thumbnails
for thumbnail in thumbnails[:download_number]:
try:
thumbnail.click()
time.sleep(1)
except Exception:
continue
# extract only the original image url because there are some urls
thumbnail_alt = thumbnail.get_attribute('alt')
images = wd.find_elements_by_css_selector('img.n3VNCb')
for image in images:
image_alt = image.get_attribute('alt')
if thumbnail_alt == image_alt and 'http' in image.get_attribute('src'):
image_urls.append(image.get_attribute('src'))
print('Success: Fetched image urls count', len(image_urls))
return image_urls
def input(value,driver: webdriver, css,index=None, describe=None):
"""
用途：输入框中输入内容
:param self:
:param value:待输入的数据
:param driver: webdriver
:param css: css选择器
:param index: 列表的下标
:param describe: 描述信息
:return:
"""
js = """var elm = document.querySelectorAll("{css}")[{index}];
elm.style.border="2px solid red";
elm.value = "{value}";""".format(css=css, index=_set_index(index), value=value)
driver.execute_script(js)
def scroll_to_end(web_driver: selenium.webdriver, scroll_pause_time=2):
# Get scroll height
last_height = web_driver.execute_script("return document.body.scrollHeight")
while True:
# Scroll down to bottom
web_driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
# Wait to load page
time.sleep(scroll_pause_time)
# Calculate new scroll height and compare with last scroll height
new_height = web_driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
def _full_screenshot_old(browser: webdriver, offset: int = 0) -> Image:
""" Получение полного скриншота содержимого окна браузера
offset - начальное смещение"""
img_list = [] # для хранения фрагментов изображения
# Определение размера окна браузера
height = browser.execute_script("""return Math.max(
document.documentElement.clientHeight,
window.innerHeight
);""")
# Определение высоты содержимого окна браузера
max_window_height = browser.execute_script("""return Math.max(
document.body.scrollHeight,
document.body.offsetHeight,
document.documentElement.clientHeight,
document.documentElement.scrollHeight,
document.documentElement.offsetHeight
);""")
# Скроллим страницу и добавляем скриншот видимого окна браузера в img_list
while offset < max_window_height:
time.sleep(1)
browser.execute_script(f"window.scrollTo(0, {offset});")
img = Image.open(BytesIO((browser.get_screenshot_as_png())))
img_list.append(img)
offset += height
# Обрезание последнего изображения
box = (0, height - height *
(max_window_height / height - max_window_height // height),
img_list[-1].size[0], img_list[-1].size[1])
img_list[-1] = img_list[-1].crop(box)
# Определение размеров нового изображения, создание холста
img_frame_height = sum([img_frag.size[1] for img_frag in img_list])
img_frame = Image.new('RGB', (img_list[0].size[0], img_frame_height))
# Объединение изображений в одно
offset = 0
for img_fragment in img_list:
img_frame.paste(img_fragment, (0, offset))
offset += img_fragment.size[1]
return img_frame
