def __init__(self):
self.date_file = data_file
self.result_file = os.getcwd() + '\\data\\' + str(
uuid.uuid4()) + '.xlsx'
self.air_url = 'https://gateway.uniner.com/track-j/api/airliftTrack/query'
self.ship_url = 'https://gateway.uniner.com/track-j/api/shippingTrack/query'
self.ups_url = 'https://gateway.uniner.com/track-j/api/expressTrack/query'
try:
self.server = subprocess.Popen(
r"./ms-playwright/chromium-907428/chrome-win/chrome.exe --remote-debugging-port=9002"
)
except:
os.popen(
'./chrome --no-sandbox --disable-gpu --disable-dev-shm-usage --use-gl=desktop --window-size=1600,1024 --remote-debugging-port=9002'
)
time.sleep(2)
playwright = sync_playwright().start()
self.browser = playwright.chromium.connect_over_cdp(
'http://localhost:9002')
self.context = self.browser.contexts[0]
self.page = self.context.pages[0]
if os.path.exists(os.getcwd() + '\\data\\') is False:
os.mkdir(os.getcwd() + '\\data\\')
else:
path_1 = os.getcwd() + '\\data\\'
list_dir = os.listdir(path_1)
for i in list_dir:
file_name = os.path.join(path_1 + i)
if os.path.isfile(file_name):
os.remove(file_name)
elif os.path.isdir(file_name):
shutil.rmtree(file_name)
def get_hrefs_imgs():
with sync_playwright() as p:
browser = p.chromium.launch(headless=True) # Включить отображение окна браузера
page = browser.new_page() # Открывает новое окно
page.goto("https://www.supremenewyork.com/shop/all") # Переходим на страницу Supreme
# Получаем объект с ссылкой на товар, а также с ссылкой на изображение
clothes = page.query_selector_all("div.inner-article")
# Инициализация списка с ссылками на товар и ссылками на изображения
lst_href_img = []
# Пройдем по каждому элементу одежды
for element in clothes:
# Разбиваем на теги
result = re.split(r' ', element.inner_html())
# Вытаскиваем ссылку на товар
str_href = 'https://www.supremenewyork.com'
str_href += result[2][6:-6]
lst_href_img.append(str_href)
# Вытаскиваем ссылку на изображение
str_href = 'https:'
str_href += result[5][5:-1]
lst_href_img.append(str_href)
# Получаем все изображения
count = 0
num = 0
for element in lst_href_img:
if count % 2 == 1:
num += 1
page.goto(element)
page.screenshot(path=f'img/{num}.png')
count += 1
browser.close()
return lst_href_img
def run(self):
with sync_playwright() as playwright:
browser = getattr(playwright, browser_name).launch()
# This should not throw ^^.
browser.new_page()
browser.close()
result.append("Success")
def run(self) -> None:
with sync_playwright() as playwright:
browser = playwright[browser_name].launch(**launch_arguments)
# This should not throw ^^.
browser.new_page()
browser.close()
result.append("Success")
def download_all_courses_videos():
p = sync_playwright().start()
browser = p.firefox.launch(headless=True)
try:
page = execute_login(browser)
enrolled_courses = [
utils.format_course(course) for course in list_courses(page)
]
if conf.selected_only and enrolled_courses:
courses = [
c for c in enrolled_courses
if any(substring in c for substring in conf.courses)
]
else:
courses = enrolled_courses
logger.info("Selected courses for download: ")
logger.info("\n".join(courses))
for course in courses:
download_course_videos(page, course)
page.close()
except Exception as e:
logger.exception(f"Error while running kadenze-dl: {e}")
finally:
browser.close()
p.stop()
def playwright_browsers(request):
if request.config.option.runner.lower() != "playwright":
yield {}
else:
# import playwright here to allow running tests without playwright installation
try:
from playwright.sync_api import sync_playwright
except ImportError:
pytest.exit(
"playwright not installed. try `pip install playwright && python -m playwright install`",
returncode=1,
)
with sync_playwright() as p:
try:
chromium = p.chromium.launch(args=[
"--js-flags=--expose-gc",
], )
firefox = p.firefox.launch()
# webkit = p.webkit.launch()
except Exception as e:
pytest.exit(f"playwright failed to launch\n{e}", returncode=1)
try:
yield {
"chrome": chromium,
"firefox": firefox,
# "webkit": webkit,
}
finally:
chromium.close()
firefox.close()
def test_sync_playwright_multiple_times():
with pytest.raises(Error) as exc:
with sync_playwright() as pw:
assert pw.chromium
assert (
"It looks like you are using Playwright Sync API inside the asyncio loop."
in exc.value.message)
def open(self):
self._pw_ctx = sync_playwright()
self.pw = self._pw_ctx.start()
log.debug(
f"launching chromium browser with kwargs: {self.browser_kwargs}")
self.browser = self.pw.chromium.launch(**self.browser_kwargs)
self.page = self.browser.new_page()
def playright_get_url(url, type="firefox", delay=0, headless=True):
type = type.lower()
if type not in TYPE_LIST:
raise Exception("Type {} is invalid.\nTYPE should only be 'chromium' 'firefox' or 'webkit'".format(type))
with sync_playwright() as p:
if type == TYPE_LIST[0]:
browser = p.chromium.launch(headless=headless)
elif type == TYPE_LIST[1]:
browser = p.firefox.launch(headless=headless)
else:
browser = p.webkit.launch(headless=headless)
try:
page = browser.new_page()
page.goto(url=url)
if delay != 0:
time.sleep(delay)
html = page.inner_html("*")
except:
raise Exception(GET_PAGE_SOURCE_ERROR_MSG)
finally:
browser.close()
return BeautifulSoup(html, 'lxml')
def lambda_handler(event, context):
"""Sample pure Lambda function
Parameters
----------
event: dict, required
context: object, required
Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Output : dict
"""
# print(os.environ)
if account is None:
raise ValueError('Value Error')
with sync_playwright() as playwright:
run(playwright)
return {
"statusCode": 200,
"body": json.dumps({
"message": "hello world",
}),
}
def test_browser(browser_name):
playwright = sync_playwright().start()
if browser_name == 'webkit':
browser = playwright.webkit.launch()
else:
browser = playwright.chromium.launch()
page = browser.new_page()
page.goto('https://www.tourney.nz/')
page.wait_for_timeout(3000) # Todo
#with page.expect_request("**/*logo*.png") as first:
# page.goto("https://wikipedia.org")
#print(first.value.url)
page.screenshot(path=browser_name + '_01_root.png')
page.goto('https://www.tourney.nz/#/')
page.wait_for_timeout(3000) # Todo
page.screenshot(path=browser_name + '_02_tournaments.png')
page.goto('https://www.tourney.nz/#/6HFv9HVjv68D7Pn2KwHtXB')
page.wait_for_timeout(3000) # Todo
page.screenshot(path=browser_name + '_03_tournament.png')
browser.close()
playwright.stop()
def test_anyName():
with sync_playwright() as p:
for browser_type in [p.chromium, p.firefox, p.webkit]:
browser = browser_type.launch()
page = browser.new_page()
page.goto('http://whatsmyuseragent.org/')
page.screenshot(path=f'example-{browser_type.name}.png')
browser.close()
async def test_titulo(self):
async with sync_playwright() as p:
browser = await p.chromium.launch()
page = await browser.new_page()
await page.goto("https://www.ucp.edu.ar/")
titulo = await page.title()
await browser.close()
self.assertEqual("Universidad de la Cuenca del Plata -", titulo)
def get_playwright():
global playwright
if playwright is None:
try:
playwright = sync_playwright().start()
except Exception as e:
raise e
return playwright
def _crawler_handler(self, url):
print('start crawling url:{}'.format(url))
results = []
def log_and_continue_request(route: Route, request: Request):
resource_type = request.resource_type
'''请求过滤'''
if resource_type in ['image', 'media', 'eventsource', 'websocket']:
route.abort()
else:
url_origin = request.url
headers = request.headers
method = request.method
post_data_json = request.post_data_json
print(url_origin, headers, method, post_data_json)
if not url_origin:
route.continue_()
return
if not self.filter_url_by_domain(url_origin) or self.filter_ext(url_origin):
route.continue_()
return
http_har = dict()
if method == 'POST' or method == 'PUT':
post_data_origin = post_data_json
post_data_handled = self._parse_post_data(post_data_origin)
content_type = headers['content-type'] if 'content-type' in headers else ''
http_har['originPostData'] = post_data_origin
http_har['postData'] = post_data_handled
http_har['contentType'] = content_type
http_har['url'] = url_origin
http_har['originUrl'] = url_origin
http_har['method'] = method
if method == 'GET':
http_har = self.parse_static_url(url_origin)
results.append(http_har)
route.continue_()
with sync_playwright() as p:
browser = p.webkit.launch(headless=True, chromium_sandbox=True, )
page = browser.new_page()
page.set_default_navigation_timeout(30000)
if self.cookie:
self.header['Cookie'] = self.cookie
page.set_extra_http_headers(headers=self.header)
page.route('**/*', log_and_continue_request)
page.goto(url)
page.wait_for_load_state(state="networkidle", timeout=30000)
tmp_link = self.static_crawler(page, results, url)
for link in tmp_link:
link.click()
self.static_crawler(page, results, url)
browser.close()
self._handle_url(results)
def __init__(self):
"""Initialize the worker object
"""
with sync_playwright() as p:
self.browser = p.chromium.launch()
self.requests_session = requests.Session()
self.requests_session.headers['User-Agent'] = DEFAULT_USER_AGENT
def test_grab(self):
with sync_playwright() as p:
# for browser_type in [p.chromium, p.firefox, p.webkit]:
for browser_type in [p.chromium]:
browser = browser_type.launch()
page = browser.new_page()
page.set_viewport_size({"width": 640, "height": 480})
page.goto('http://whatsmyuseragent.org/')
page.screenshot(path=f'example-{browser_type.name}.png')
browser.close()
def test_correct_email(self):
email = '*****@*****.**'
with sync_playwright() as p:
browser = p.chromium.launch()
page = browser.new_page()
page.goto('https://dev.lk.tr-line.ru/restore-password')
page.type("input[name=forget_email]", email)
page.click('text=Сбросить пароль')
sleep(0.5)
res = page.query_selector('//html/body/div[2]')
self.assertTrue(res is not None)
def test_events(browser_name: str, launch_arguments: Dict) -> None:
with pytest.raises(Exception, match="fail"):
def fail() -> None:
raise Exception("fail")
with sync_playwright() as p:
with p[browser_name].launch(**launch_arguments) as browser:
with browser.new_page() as page:
page.on("response", lambda _: fail())
page.goto("https://example.com")
def najave(datum_in, vrijeme_boravka_in, ulaz_in, izlaz_in, grupe_odraslih,
grupe_djece, grupa_djece_0_7):
with sync_playwright() as playwright:
browser = playwright.chromium.launch(headless=False)
context = browser.new_context()
page = context.new_page()
page.goto("http://oa.rao.hr:8050/#/eticket/partner-web")
page.fill("input[type=\"text\"]", "username")
page.fill("input[type=\"password\"]", "password")
page.click("text=Prijava")
with page.expect_navigation():
page.click("img[alt=\"NP KRKA\"]")
page.goto(
"http://oa.rao.hr:8050/#/eticket/partner-web/nova-najava-krka")
page.click("text=Odaberite klijenta")
page.click("#top-section >> text=VRATA KRKE D.O.O.")
page.click("text=Datum dolaska")
page.click(f"[aria-label=\"{datum_in}\"]")
page.click("text=Odaberite ulaz")
page.click(f"text={ulaz_in}")
page.click("text=Odaberite nacin ulaska u park")
page.click("text=Pjesaci")
page.click("text=Odaberite izlaz")
page.click(f":nth-match(:text(\"{izlaz_in}\"), 3)")
page.click("text=Odaberite nacin izlaska iz parka")
page.click(":nth-match(:text(\"Pjesaci\"), 3)")
page.click("text=Dohvati periode ulaska")
page.click(f"text={datum_in} {vrijeme_boravka_in}")
page.click(
f"text={datum_in} 18:00 07.09.2021 19:00 IZLAZ 969 Pjesaci >> td")
if grupe_odraslih:
page.click("text=Dodaj ulaznicu")
page.click("text=Grupe odraslih")
page.fill(
"text=Grupe odraslih (147 kn) Količina 1 x 147 kn = 147 kn Dodaj u košaricu >> input[type=\"text\"]",
f"{grupe_odraslih}")
page.click("text=Dodaj u košaricu")
if grupe_djece:
page.click("text=Dodaj ulaznicu")
page.click("text=Grupe djece")
page.fill(
"text=Grupe djece (58.8 kn) Količina 1 x 58.8 kn = 58.8 kn Dodaj u košaricu >> input[type=\"text\"]",
f"{grupe_djece}")
page.click("text=Dodaj u košaricu")
if grupa_djece_0_7:
page.click("text=Dodaj ulaznicu")
page.click("text=Grupa djece 0-7")
page.fill(
"text=Grupa djece 0-7 (0 kn) Količina 1 x 0 kn = 0 kn Dodaj u košaricu >> input[type=\"text\"]",
f"{grupa_djece_0_7}")
page.click("text=Dodaj u košaricu")
context.close()
browser.close()
def test_events(browser_name: str, launch_arguments: Dict,
server: Server) -> None:
with pytest.raises(Exception, match="fail"):
def fail() -> None:
raise Exception("fail")
with sync_playwright() as p:
with p[browser_name].launch(**launch_arguments) as browser:
with browser.new_page() as page:
page.on("response", lambda _: fail())
page.goto(server.PREFIX + "/grid.html")
def initBrowser(browserType="chromium", browserConfig={"headless": False}):
"""
For playwright, init to create a browser. For later use, such as google search
Args:
browserType (str): Playwright browser type: chromium / firefox / webkit
browserConfig (dict): Playwright browser config. Default is {"headless": False}.
Returns:
BrowserType
Examples:
browserConfig
{
"headless": False,
"proxy": {
"server": "http://127.0.0.1:58591",
}
}
Raises:
"""
curBrowserType = None
if browserType:
browserType = browserType.lower()
# with sync_playwright() as p:
# p = sync_playwright()
p = sync_playwright().start()
# 多次调用，会：
# 发生异常: Error
# It looks like you are using Playwright Sync API inside the asyncio loop.
# Please use the Async API instead.
if browserType == "chromium":
curBrowserType = p.chromium
elif browserType == "firefox":
curBrowserType = p.firefox
elif browserType == "webkit":
curBrowserType = p.webkit
print("curBrowserType=%s" % curBrowserType)
# curBrowserType=<BrowserType name=chromium executable_path=/Users/limao/Library/Caches/ms-playwright/chromium-901522/chrome-mac/Chromium.app/Contents/MacOS/Chromium>
if not curBrowserType:
print("Unsupported playwright browser type: %s" % browserType)
return None
# browser = curBrowserType.launch(headless=False)
# browser = curBrowserType.launch(**browserLaunchOptionDict)
browser = curBrowserType.launch(**browserConfig)
print("browser=%s" % browser)
# browser=<Browser type=<BrowserType name=chromium executable_path=/Users/limao/Library/Caches/ms-playwright/chromium-901522/chrome-mac/Chromium.app/Contents/MacOS/Chromium> version=93.0.4576.0>
return browser
def resolve_screenshot(self, info, url, width, height):
with sync_playwright() as p:
browser_type = p.chromium
browser = browser_type.launch()
page = browser.new_page()
page.set_viewport_size({"width": width, "height": height})
page.goto(url)
img_byte = page.screenshot(full_page=True)
# encoded_img = "data:image/png;base64," + base64.b64encode(
# img_byte).decode()
encoded_img = base64.b64encode(img_byte).decode()
browser.close()
return encoded_img
def test_correct_user_login_sms(self):
phone = '+79214228036'
with sync_playwright() as p:
browser = p.chromium.launch()
page = browser.new_page()
page.goto('https://dev.lk.tr-line.ru/sign-in')
page.type("input[name=login]", phone)
page.click('//html/body/div/div/div[2]/main/div/div/div/div/div/div/div/div[2]/div/div/div[2]/button[1]')
sleep(0.5)
page.screenshot(path='ex.png')
res = page.query_selector('//html/body/div[2]/div')
self.assertTrue(res.inner_text() == 'Сообщение отправлено' or
'превышен лимит' in res.inner_text().lower())
def test_correct_user_login(self):
email = '*****@*****.**'
password = '******'
with sync_playwright() as p:
browser = p.chromium.launch()
page = browser.new_page()
page.goto('https://dev.lk.tr-line.ru/sign-in')
page.type("input[name=login]", email)
page.type("input[name=password]", password)
page.click('//html/body/div/div/div[2]/main/div/div/div/div/div/div/div/div[2]/div/div/div[2]/button[2]')
sleep(0.5)
res = page.query_selector('//html/body/div[2]/div')
self.assertTrue(res is None)
def cge_session():
p = sync_playwright().start()
browser_type = p.chromium
browser = browser_type.launch(headless=True) #,slow_mo=500
# Setup directory to store videos of runs
# page = browser.new_page(record_video_dir="videos/")
page = browser.new_page()
yield page
page.close()
p.stop()
def generate_browser(self, engine, browser_type='chromium'):
with sync_playwright() as p:
browser = {
'chromium': p.chromium,
'firefox': p.firefox,
'webkit': p.webkit
}
self.__browser = browser[browser_type]
self.__browser_type = browser_type
self.open_browser()
self.open_page()
engine.scheduler()
self.close()
def main():
config = Settings()
with sync_playwright() as p:
browser = p.chromium.launch()
page = browser.new_page()
try:
page.goto("https://selfservice.udi.no/en-gb/")
page.click("#ctl00_BodyRegion_PageRegion_MainRegion_LogInHeading")
page.type("input[type=email]", config.EMAIL)
page.type("input[type=password]", config.PWD.get_secret_value())
page.click("#next")
except PlaywrightTimeoutError:
msg = "Seems like UDI website is down or you are offline"
print(msg)
telegram_send.send(messages=[msg])
return
try:
book_btn_id: str = "#ctl00_BodyRegion_PageRegion_MainRegion_IconNavigationTile2_heading"
page.wait_for_selector(book_btn_id)
page.click(book_btn_id)
except playwright.helper.TimeoutError:
msg = "Failed to login. Check your password."
print(msg)
telegram_send.send(messages=[msg])
return
# click on the first one in the list
page.click(
"#ctl00_BodyRegion_PageRegion_MainRegion_ApplicationOverview_applicationOverviewListView_ctrl0_btnBookAppointment"
)
try:
page.wait_for_selector(
"#ctl00_PageRegion_MainContentRegion_ViewControl_spnReceiptAndBooking_divErrorMessageForNoAvailabelAppointments",
timeout=5000,
)
print("No appointments")
return
except PlaywrightTimeoutError:
msg = "Looks like UDI is ready for appointments"
telegram_send.send(messages=[msg])
with tempfile.TemporaryFile("r+b") as fp:
encoded_img = page.screenshot(type="png")
fp.write(encoded_img)
fp.seek(0, 0)
telegram_send.send(images=[fp])
def login_site():
with sync_playwright() as p:
browser = p.chromium.launch_persistent_context(
user_data_dir=os.path.join(os.path.dirname(__file__), 'user_data'), headless=False)
page = browser.new_page()
# entra na pagina do anime
page.goto('https://pt.ucoin.net/')
page.click('text=Entrar')
page.fill('input[name="login-email"]', '*****@*****.**')
page.fill('input[name="login-passwd"]', 'Chr1571@n')
page.click('input[name="remember"]')
page.click('button[type="submit"]')
page.close()
def __init__(self) -> None:
self.session = requests.Session()
self.playwright = sync_playwright().start()
if self.browserType == "firefox":
self.browser = self.playwright.firefox.launch(
headless=self.headless)
else:
self.browser = self.playwright.webkit.launch(
headless=self.headless)
self.page = self.browser.new_page(user_agent=USER_AGENT,
viewport=VIEWPORT)
stealth_sync(self.page)
