def __init__(self): self.in_aws = os.environ.get("AWS_EXECUTION_ENV") is not None self._driver = WebDriverWrapper(download_location="/tmp") self._parameters = SSMParameterStore(prefix="/prod")["crunchtime"] self._driver = WebDriverWrapper(download_location="/tmp") return
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() driver.get_url(week_data['es_url']) driver.get_soup() es_data.ProcessGamesToS3(driver.soup, week_id) driver.close() driver = WebDriverWrapper() driver.get_url(week_data['os_url']) driver.get_soup() os_data.ProcessGamesToS3(driver.soup) driver.close()
def __init__(self, day): self.driver = WebDriverWrapper()._driver self.wait = WebDriverWait(self.driver, 10) #Diario ou retroativo self.scrape(day)
def lambda_handler(event, context): driver = WebDriverWrapper() driver.get_url('http://35.236.100.236:8080') number = int(random.random()*100)%4 if number == 0: driver.click("//span[@data-customer='123']") elif number == 1: driver.click("//span[@data-customer='392']") elif number == 2: driver.click("//span[@data-customer='731']") else: driver.click("//span[@data-customer='567']") print("--------------------------") print("Success") print("--------------------------") time.sleep(1) driver.close() return { 'statusCode': 200, 'body': json.dumps({'message': 'Success - Clicked HotRod link'+' '+str(number)}) }
def get_all_super_markets(): driver = WebDriverWrapper() driver.get_url('https://www.tesco.com/groceries/en-GB/products/296734865') html = driver.print_all tesco_price = get_prices( "tesco", "https://www.tesco.com/groceries/en-GB/products/296734865", "span", "value", html) asda_price = get_prices( "asda", "https://groceries.asda.com/product/pringles-tube-snacks/pringles-original-sharing-crisps/910003062100", "strong", "co-product__price pdp-main-details__price", driver) morrisons_price = get_prices( "morrisons", "https://groceries.morrisons.com/products/pringles-original-372817011", "h2", "bop-price__current", driver) sainsburys_price = get_prices( "sainsburys", "https://www.sainsburys.co.uk/gol-ui/product/pringles-original-190g", "div", "pd__cost__total undefined", driver) coop_price = get_prices( "coop", "https://www.coop.co.uk/products/pringles-original-200g", "p", "coop-c-card__price", driver) driver.close list_of_shop_prices = [ tesco_price, asda_price, morrisons_price, sainsburys_price, coop_price ] return list_of_shop_prices
def get_sgd_price(): html = None url = 'https://www.rba.gov.au/statistics/frequency/exchange-rates.html' selector = 'SGD' delay = 5 # sec, to wait for item to load price = 0 # default value driver = WebDriverWrapper() driver._driver.get(url) try: # wait until element visible WebDriverWait(driver._driver, delay).until( EC.presence_of_element_located((By.ID, selector))) except TimeoutException: print('Loading took too long.') else: html = driver._driver.page_source finally: driver.close() if html: soup = bs4.BeautifulSoup(html, "html.parser") data = soup.find(attrs={"id": selector}) price = data.contents[3].next return str(price) # str for dynamoDB
def get_oil_price(): html = None url = 'https://www.tradingview.com/symbols/NYMEX-AV02%21/' selector = 'tv-symbol-price-quote__value js-symbol-last' selector1 = 'tv-symbol-price-quote__value' # use this to check when javascript on page has loaded delay = 5 # sec, to wait for item to load price = 0 # default value driver = WebDriverWrapper() driver._driver.get(url) try: # wait until element visible WebDriverWait(driver._driver, delay).until( EC.presence_of_element_located((By.CLASS_NAME, selector1))) except TimeoutException: print('Loading took too long.') else: html = driver._driver.page_source finally: driver.close() if html: soup = bs4.BeautifulSoup(html, "html.parser") data = soup.find("div", class_=selector) price = data.text return str(price) # str for dynamoDB
def scrape_amazon_reviews(config): """ Scrapes amazon reviews. """ urls = config['urls'] sleep_range = (1, 3) driver = WebDriverWrapper() try: for i, product_url in enumerate(urls): driver.open_amazon_product(product_url) driver.scrape_reviews() # sleep a bit if sleep_range: random_sleep(sleep_range) except AmazonDetectionException as e: logger.fatal('Amazon detected the scraping. Aborting.') pass pprint.pprint(driver.results) driver.close() logger.info('Got {} results out from {} urls'.format(len(driver.results.keys())-3, len(urls))) return driver.results, driver.status
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() driver.get_url('http://example.com') example_text = driver.get_inner_html('(//div//h1)[1]') driver.close() return example_text
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() driver.get_url('https://www.gta-homes.com/') # body = driver.get_body() # print(body) driver.close()
def scraper_handler(event, context): driver = WebDriverWrapper() driver.get_url(event['url']) source = driver.get_html() driver.close() return source
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() receiptnumber = os.getenv("RECEIPTNUMBER") driver.get_url('https://egov.uscis.gov/casestatus/landing.do') driver.set_input_value( '/html/body/div[2]/form/div/div[1]/div/div[1]/fieldset/div[1]/div[4]/input', receiptnumber) driver.click( '/html/body/div[2]/form/div/div[1]/div/div[1]/fieldset/div[2]/div[2]/input' ) example_text = driver.get_inner_html('(//div//h1)[1]') driver.close() return example_text
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() example_text = '' sekinAl = "https://seekingalpha.com/market-news/all" # driver.get("http://www.python.org") driver.get_url(sekinAl) # assert "Python" in driver.title # driver elem = driver.find_elements_by_class_name("title") for i in elem: print(i.text) driver.close() return example_text
def _login(self): self._driver = WebDriverWrapper(download_location="/tmp") driver = self._driver._driver driver.implicitly_wait(25) driver.set_page_load_timeout(45) driver.get("https://partner.postmates.com/dashboard/home/payments") driver.find_element_by_xpath('//input[@name="email"]').send_keys( self._parameters["user"]) driver.find_element_by_xpath('//input[@name="password"]').send_keys( self._parameters["password"] + Keys.ENTER) sleep(3) driver.get("https://partner.postmates.com/dashboard/home/payments") sleep(2) return
def _login(self): self._driver = WebDriverWrapper(download_location="/tmp") driver = self._driver._driver driver.implicitly_wait(25) driver.set_page_load_timeout(45) driver.get( "https://restaurant.grubhub.com/financials/deposit-history/1669366/" ) driver.find_elements_by_xpath("//input")[0].send_keys( self._parameters["user"]) driver.find_elements_by_xpath("//input")[1].send_keys( self._parameters["password"] + Keys.ENTER) sleep(4) return
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() driver.get_url('https://www.backerkit.com/admins/sign_in') driver.set_input_value_byName('admin[email]', 'YOUR_EMAIL') driver.set_input_value_byName('admin[password]', 'YOUR_PASSWORD') driver.click_byName('commit') driver.get_url('URL_OF_EXPORT_PAGE_OF_DESIRED_SEGMENT') driver.set_input_value('XPATH_OF_FILE', '\n') driver.close()
def _login(self): self._driver = WebDriverWrapper(download_location="/tmp") driver = self._driver._driver driver.implicitly_wait(25) driver.set_page_load_timeout(45) driver.get("https://fms.flexepos.com/FlexeposWeb/") driver.find_element_by_id("login:username").clear() driver.find_element_by_id("login:username").send_keys( self._parameters["user"]) driver.find_element_by_id("login:password").clear() driver.find_element_by_id("login:password").send_keys( self._parameters["password"]) driver.find_element_by_name("login:j_id29").click() return
def _login(self): self._driver = WebDriverWrapper(download_location="/tmp") driver = self._driver._driver driver.implicitly_wait(25) driver.set_page_load_timeout(45) driver.get( "https://merchant-portal.doordash.com/merchant/financials?store_id=631548" ) driver.find_element_by_xpath( '//input[@data-anchor-id="IdentityLoginPageEmailField"]' ).send_keys(self._parameters["user"]) driver.find_element_by_xpath( '//input[@data-anchor-id="IdentityLoginPagePasswordField"]' ).send_keys(self._parameters["password"]) driver.find_element_by_id("login-submit-button").click() return
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() driver.get_url('https://www.google.es/') driver.set_input_value('//input[@name="q"]', '21 buttons') button = driver.find("(//input[@name='btnK'])[2]") button.send_keys(Keys.TAB) driver.click('//input[@name="btnK"]') first_google_result_title = driver.get_inner_html( '(//div[@class="rc"]//a)[1]') print("--------------------------") print(first_google_result_title) print("--------------------------") driver.close()
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() driver.get_url('https://www.myprotein.com/nutrition/protein.list') # driver.set_input_value('//input[@name="q"]', '21 buttons') # button = driver.find("(//input[@name='btnK'])[2]") # button.send_keys(Keys.TAB) # driver.click('//input[@name="btnK"]') time.sleep(2) offers_html = driver.get_inner_html( "//*[@class='stripBanner']//*[@class='stripBanner_text']//p") print("--------------------------") print('Banner element:', offers_html) voucher_code_reg = re.search(r"CODE\:(.*)", offers_html) # print("Code (match):", voucher_code_reg) print("Code (regex):", voucher_code_reg.group(0)) voucher_code = voucher_code_reg.group(1).strip() print("Code:", voucher_code) voucher_text_reg = re.search(r"(.*)\|", offers_html) print("Code desc (regex):", voucher_text_reg.group(0)) voucher_desc = re.sub('<[^<]+?>', ' ', voucher_text_reg.group(1).strip()) print("Code description:", voucher_desc) print("--------------------------") # first_google_result_title = driver.get_inner_html( # '(//div[@class="rc"]//a)[1]') # print("--------------------------") # print(first_google_result_title) # print("--------------------------") driver.close() return {'message': f"Code: {voucher_code}, Description: {voucher_desc}"}
def ProcessUrl(url, width, height): currentOrientation = "landscape" if width >= 1000: currentOrientation = "portrait" width, height = DetermineSizeFromOrientation(currentOrientation) try: # Start up the web driver. driver = WebDriverWrapper(str(width), str(height)) driver.get_url(url) # Allow the dashboards to load, to allow the web report sections done increment or decrement the counter time.sleep(3000) WebDriverWait() wait = WebDriverWait(driver, 32) isReady = False while not isReady: try: wait.Until(driver.execute_script('return window.webReportSectionsDone')) except WebDriverTimeoutException as e: # Catch timeout exceptions here as this means that at least one of the sections times out while loading. # If one section times out in a report, the entire report should not be generated. #cancellationToken.ThrowIfCancellationRequested(); #except InvalidOperationException as e: # Catch invalid operation exception if the page is not loaded yet but code need to access 'webReportSectionsDone' isReady = driver.execute_script('return window.webReportSectionsDone') #cancellationToken.ThrowIfCancellationRequested(); # Get the page source outputCDontents = driver.page_source #TODO: driver.Navigate().GoToUrl(new Uri(url).GetLeftPart(UriPartial.Authority) + "/#/logout"); #Initiate a logout request so the next time we use this instance, we don't reuse the session parsed_uri = urlparse(url) result = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri) driver.get_url(result + '#/logout') return outputCDontents finally: driver.close()
def _login(self): self._driver = WebDriverWrapper(download_location="/tmp") driver = self._driver._driver driver.implicitly_wait(25) driver.set_page_load_timeout(45) driver.get( "https://restaurant.uber.com/v2/payments?restaurantUUID=8d6b329b-4976-4ef7-8411-3a416614a726" ) driver.find_element_by_id("useridInput").send_keys( self._parameters["user"] + Keys.RETURN) driver.find_element_by_id("password").send_keys( self._parameters["password"] + Keys.RETURN) sleep(5) for element, pin in zip(driver.find_elements_by_xpath("//input"), self._parameters["pin"]): element.send_keys(pin) driver.find_element_by_xpath("//button").click() sleep(10) return
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() driver.get_url('https://www.google.es/') driver.set_input_value('//input[@id="lst-ib"]', '21 buttons') driver.click('//center//img[@alt="Google"]') time.sleep(0.5) driver.click('//input[@name="btnK"]') time.sleep(0.5) first_google_result_title = driver.get_inner_html( '(//div[@class="rc"]//a)[1]') print("--------------------------") print(first_google_result_title) print("--------------------------") driver.close()
def lambda_handler(event, context): driver = WebDriverWrapper() bucket = event['bucket'] html_key = event['task']['html_key'] #dataset_key = event['task']['dataset_key'] #config_key = event['task']['config_key'] png_key = event['task']['png_key'] #open('/tmp/datasets.js', 'w').write(get_file(bucket, dataset_key)) #open('/tmp/mapConfig.js', 'w').write(get_file(bucket, config_key)) driver.get_url(get_html(bucket, html_key)) time.sleep(15) driver.get_screenshot_as_file('/tmp/heatmap.png') print(driver._driver.capabilities) s3.upload_file('/tmp/heatmap.png', Bucket=bucket, Key=png_key) driver.close()
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() example_text = '' sekinAl="https://seekingalpha.com/market-news/all" # driver.get("http://www.python.org") driver.get(sekinAl) # assert "Python" in driver.title # driver elem = driver.find_elements_by_class_name("title") for i in elem: if 'doge' in i.text print(i.text) # /html/body/div[3]/div/div/div[2]/ul/li[2]/div[2]/div[1] # elem.clear() # elem.send_keys("pycon") # elem.send_keys(Keys.RETURN) # assert "No results found." not in driver.page_source driver.close() return example_text
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper() driver.get_url('https://www.amazon.com/') html = driver.get_inner_html('//*[@id="nav-your-amazon"]') # driver.click('//center//img[@alt="Google"]') # time.sleep(0.5) # # driver.click('//input[@name="btnK"]') # time.sleep(0.5) # # first_google_result_title = driver.get_inner_html('(//div[@class="rc"]//a)[1]') print("--------------------------") # print(first_google_result_title) print(html) print("--------------------------") driver.close() return 0
def __create_driver(self) -> WebDriverWrapper: return WebDriverWrapper(self.__driver_path)
from multiprocessing import Process, Pipe from multiprocessing.pool import ThreadPool import threading # import pandas as pd # threadLocal = threading.local() # Global array # big = [] # Global response responses = [] # WebDriver browser = WebDriverWrapper() driver = browser._driver def s3_handler(full_path, data): s3 = boto3.client('s3') bucket = 'freshket-marketprice' # csv_buffer = StringIO() # df.to_csv(csv_buffer) uploadByteStream = json.dumps(data, indent=4, sort_keys=True, default=str) # uploadByteStream = bytes(json.dumps(data).encode('UTF-8'))efault).encode('UTF-8') # response = s3.put_object(Bucket=bucket, Key=fileName, Body=csv_buffer.getvalue()) response = s3.put_object(Bucket=bucket, Key=full_path, Body=uploadByteStream) return response def parsing(category_name):
import time from webdriver_wrapper import WebDriverWrapper from selenium.webdriver.common.keys import Keys import time import json import boto3 import pandas as pd if __name__ == "__main__": driver = WebDriverWrapper() driver.get_url('https://www.google.com/') page_title = driver.get_page_title() print("--------------------------") print(page_title) print("--------------------------") driver.close() data = {} data['page_title'] = page_title df = pd.DataFrame(data) df.to_csv("s3://freshket-marketprice/test.csv", index=False)
def lambda_handler(*args, **kwargs): driver = WebDriverWrapper()._driver db = firebase.FirebaseApplication( 'https://dash-associados-default-rtdb.firebaseio.com/', None) data = {"username": "******"} db.post('/users', data) # driver.get("https://associados.amazon.com.br") # driver.get_url('http://example.com') # example_text = driver.get_inner_html('(//div//h1)[1]') # driver.close() driver.get("https://associados.amazon.com.br") try: cookies_file = open("cookies.txt") if os.fstat(cookies_file.fileno()).st_size == 0: raise IOError for cookie in cookies_file: driver.add_cookie(json.loads(cookie)) except IOError: driver.find_element_by_xpath("//a[@href='/login']").click() username = driver.find_element_by_id("ap_email") username.clear() username.send_keys("*****@*****.**") password = driver.find_element_by_id("ap_password") password.clear() password.send_keys("infOaz19!") driver.find_element_by_id("signInSubmit").click() while ('home' not in driver.current_url): if 'approval' in driver.current_url: print('Aprove o login no celular.') fastrack = WebDriverWait(driver, 300).until( ec.visibility_of_element_located( (By.XPATH, "//div[@data-assoc-eid='ac-home-month-summary']"))) elif 'signin' in driver.current_url: captcha_img = driver.find_element_by_xpath( "//img[@alt='CAPTCHA']").get_attribute("src") print(captcha_img) captcha_input = driver.find_element_by_id("auth-captcha-guess") captcha = input("Digite o CAPTCHA e aperte ENTER\n") print(f'Usando o captcha "{captcha}"') captcha_input.send_keys(captcha) password = driver.find_element_by_id("ap_password") password.clear() password.send_keys("infOaz19") driver.find_element_by_id("signInSubmit").click() with open("cookies.txt", "w") as cookies_file: for cookie in driver.get_cookies(): cookies_file.write(json.dumps(cookie) + '\n') finally: cookies_file.close() summaries = driver.find_elements_by_xpath( "//div[@data-assoc-eid='ac-home-month-summary']//div[contains(@class, 'a-row')]//div[contains(@class, 'a-ws-span-last')]" ) total_sent = summaries[0].text total_gains = summaries[1].text total_ordered = summaries[2].text total_clicks = summaries[3].text driver.close() return f'Produtos pedidos: "{total_sent}" - Ganho: "{total_gains}" - Produtos pedidos: "{total_ordered}" - Cliques: "{total_clicks}"'