def scrape(self, **card): session = requests.Session() session.headers.update({ "user-agent": config.USER_AGENT, "accept": "*/*", "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en;q=0.9", }) params = {"giftCardNumber": card["number"], "pin": card["pin"]} logger.info(f"Fetching balance") resp = session.get(self.website_url, params=params) if resp.status_code != 200: raise RuntimeError( f"Failed to retrieve card balance (status code {resp.status_code})" ) # Tried to use BS4 but it refused to work, I think HTML returned was too messy/non-compliant page_parsed = lxml.html.fromstring(resp.text) try: avail_balance = page_parsed.xpath( "//div[@class='cardPoints']/div")[0].text except: # error on screen error_text = page_parsed.xpath( "//div[contains(@class,'error')]")[0].text raise RuntimeError(error_text) logger.info(f"Success! Card balance: {avail_balance}") return {"balance": avail_balance}
def check_balance(self, cards_chunk): for card in cards_chunk: if not self.validate(card): raise RuntimeError( f'Card format of {card["card_number"]} failed validation') logger.info(f"Checking balance for cards in chunk") return self.scrape(cards_chunk)
def check_balance(self, **kwargs): if self.validate(kwargs): logger.info("Checking balance for card: {}".format( kwargs["card_number"])) return self.scrape({ "accountNumber": kwargs["card_number"], "cv2": kwargs["cvv"] })
def check_balance(self, **kwargs): if self.validate(kwargs): logger.info(f"Checking balance for card: {kwargs['card_number']}") form_inputs = { "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$CardNumberTextBox": kwargs["card_number"], "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$PinTextBox": kwargs["pin"], } return self.scrape(form_inputs)
def check_balance(self, **kwargs): if self.validate(kwargs): logger.info("Checking balance for card: {}, exp {}/{}".format( kwargs["card_number"], kwargs["exp_month"], kwargs["exp_year"])) return self.scrape({ "CardNumber": kwargs["card_number"], "ExpirationDateMonth": kwargs["exp_month"], "ExpirationDateYear": kwargs["exp_year"], "SecurityCode": kwargs["cvv"], })
def check_balance(self, **kwargs): if self.validate(kwargs): logger.info("Checking balance for card: {}, exp {}/{}".format( kwargs["card_number"], kwargs["exp_month"], kwargs["exp_year"])) return self.scrape({ "cardNumber": kwargs["card_number"], "expMonth": kwargs["exp_month"], "expYear": kwargs["exp_year"], "cvv": kwargs["cvv"], })
def check_balance(self, **kwargs): if self.validate(kwargs): logger.info("Checking balance for card: {}".format( kwargs["card_number"])) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) return loop.run_until_complete( self.scrape({ "card_number": kwargs["card_number"], "pin": kwargs["pin"], }))
def check_balance(self, **kwargs): if self.validate(kwargs): logger.info("Checking balance for card: {}, exp {}/{}".format( kwargs["card_number"], kwargs["exp_month"], kwargs["exp_year"])) return self.scrape({ "number-1": kwargs["card_number"], "valid-mm": str(int(kwargs["exp_month"]) ), # Lazy way to strip '0' prefix, if present "valid-yy": "20{}".format(kwargs["exp_year"]), "pin": kwargs["cvv"], })
def scrape(self, **kwargs): # Site key gathered from https://secure2.homedepot.com/mycheckout/assets/react/giftcard.bundle.js?v=v1.2040.2 # 6LfEHBkTAAAAAHX6YgeUw9x1Sutr7EzhMdpbIfWJ and 6Le3GRkTAAAAAPpXON0jcJCLrYZnm-ZqyLhbCLbX captcha_resp = captcha_solver.solve_recaptcha( self.website_url, "6LfEHBkTAAAAAHX6YgeUw9x1Sutr7EzhMdpbIfWJ" ) if captcha_resp["errorId"] != 0: raise RuntimeError( f"Unable to solve reCAPTCHA ({captcha_resp['errorDescription']})" ) # Begin API request session = requests.Session() # Get balance check page to grab cookies session.get(website_url) session.headers.update( { "origin": "https://secure2.homedepot.com", "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", "content-type": "application/json", "accept": "application/json, text/plain, */*", "referer": "https://secure2.homedepot.com/mycheckout/giftcard", "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en;q=0.9", #'cookie': "HD_DC=origin; check=true; _abck=3D7AEA27759869A787304AD8C0B93D1C17C532E781200000EB2DB15ACADB6E34~0~Ie8mODWyYhnd9Jka+w+AC39XvfEAlhz78nuYgOCT8so=~-1~-1; AMCVS_F6421253512D2C100A490D45%40AdobeOrg=1; thda.u=5f480787-7953-8e2e-14c3-0b96cfd48040; thda.s=054d35ff-11c7-6b16-75cd-bbcd2d48c17b; RES_TRACKINGID=85286815566469932; ftr_ncd=6; thda.m=81084504873679911179154344636083105898; LPVID=Q5OGUwNTliM2EzY2NmM2Yw; cto_lwid=2dc94469-69f2-49af-886c-69251e0ee8da; THD_FORCE_LOC=1; THD_MCC_ID=6f1d76fc-8b8e-45be-97ea-9c061f79f851; cart_activity=8777201b-f4c7-4ccc-8e3e-428c62d3c730; WORKFLOW=LOC_HISTORY_BY_IP; THD_INTERNAL=0; og_session_id=379dc8f09a4311e7806bbc764e106cf4.731928.1538275068; IR_gbd=homedepot.com; ecrSessionId=6ED9C65DD7DE929C82CAEB9D58E38D71; THD_USER=\"eyJzdm9jQ3VzdG9tZXJBY2NvdW50SWQiOiIwM0VEMjFBQ0JDNTgzNTkwMFMiLCJsb2dvbklkIjoiaG9tZWRlcG90QGJpa2VtYW5kYW4uY29tIiwidXNlcklkIjoiMDUxM0UyMTkyOTIxRTY5MTBVIiwiY3VzdG9tZXJUeXBlIjoiQjJDIn0=\"; THD_USER_SESSION=AQIC5wM2LY4SfcxJD0XieCwTVsVkLiR7a2oDQMG6d90y3To.*AAJTSQACMDIAAlNLABM2MzI5MDk2NzEyMTU1OTY3NjQ4AAJTMQACMDY.*; THD_CACHE_NAV_PERSIST=\"\"; THD_CACHE_NAV_SESSION=C20%7E0_%7EC20_EXP%7E_%7EC22%7E641_%7EC22_EXP%7E_%7EC26%7EP_REP_PRC_MODE%7C0_%7EC26_EXP%7E; ak_bmsc=C6D4C7801E4881DEBB1E03DF88F8F1CF17C532D4FC2700009D15395C49757425~plfpoGCv+yaNOcuoWld8wCQ3wGC2Yh+Dl/XvPv5aTAd9s7O3guYCefrdJKkL4e+oVw8WlPV3onzF+qQyqCQ0fQrP9dWjPPDzi2FDN8mCAQ4GC4752Ucds9YyV7Vb8oOB4KiekG+BZiyaPSse/+QTkx7VA+q2wy5Kc7bTXGsK1BJKZ8nZf8sBcVVuypxX8ZJjK3x4h6ChD396S6SWGu9KOR5Gws+4xT6nE1xBZfzVt/pSA=; bm_sz=993B9150C6357D1453561ED8533D948C~QAAQ1DLFF1EGQC9oAQAAWW78Pvljdn9YU6uwR/wj13nNtKbJAd1tz/XtkSB4D0GzZ0sB6xr9f6lmedf6cPJTA7GFYjVV7H4/3px/zWyNeY73uc1wGoIs4b2EDW9ft+53/wIzZavesl7pmzhBMiCA5GKqJ4WFF9oO4+EP3gtOxVKIKMDjVP8J/UAn2a3coAOZ0zs=; THD_SESSION=; AMCV_F6421253512D2C100A490D45%40AdobeOrg=-894706358%7CMCIDTS%7C17908%7CMCMID%7C81084504873679911179154344636083105898%7CMCAID%7CNONE%7CMCOPTOUT-1547252157s%7CNONE%7CvVersion%7C2.3.0; THD_PERSIST=C4%3D641%2BRohnert%20Pk%20-%20Rohnert%20Park%2C%20CA%2B%3A%3BC4_EXP%3D1578780959%3A%3BC24%3D94928%3A%3BC24_EXP%3D1578780959%3A%3BC34%3D32.1%3A%3BC34_EXP%3D1547331358%3A%3BC39%3D1%3B7%3A00-20%3A00%3B2%3B6%3A00-22%3A00%3B3%3B6%3A00-22%3A00%3B4%3B6%3A00-22%3A00%3B5%3B6%3A00-22%3A00%3B6%3B6%3A00-22%3A00%3B7%3B6%3A00-22%3A00%3A%3BC39_EXP%3D1547248559; ResonanceSegment=1; signInBubble=true; s_dfa=homedepotprod%2Chomedepotglobaldev; ats-cid-AM-141099-sid=46341048; LPSID-31564604=tgZo3ZtbTmaR_Y-e3nvjcQ; bm_mi=21A83C44D89B199539D14330195C2F99~mGutnDw9T1dVaFjCLHwYLZw3kAOjDjotPSLR4WpSqqHiFQpXWyR4sK+9B3XaCHm6PTzxe40CC15/SSfwKqFR69zJ9MNcL+Ikczcr4Mq4STJ8E4DGF8ozcpOICIx7wUHr4bN9S6EoHVGMsDpO6tntSLxWq4i2U3SDGUXpFjyWEHVYJoMil8dK0D1+BduiBRPiryGLnSu00wtXOnXDXIgSEETU/dWSRtwRyDwYSIA02+JU+MGUWuObfgriro9Zd2tl; bm_sv=176F5E5B2FBE5E8295D8CCE4214111A6~lvtdTZIv6RPPE+bbvi5sIhT//R4DBFahLwcKcw/HA61WomPuVKOI4uq9Fr0BJTsx7o1U57Vsw3iSq+JvBAadGMOiQjWILK6R/pWM/LHXLaRQCnWoMbQbe1Is1Zc4ZKe4wStAcGNCCi2+WT4udxpsK3vuYFnaBUlGjr1uTmEGZ/o=; forterToken=2a23633194f246a7871e5c4f3f1546d8_1547247463626_21_UDF43_6; s_pers=%20s_nr%3D1547247468412-Repeat%7C1578783468412%3B%20s_dslv%3D1547247468416%7C1641855468416%3B%20s_dslv_s%3DLess%2520than%25201%2520day%7C1547249268416%3B%20productnum%3D98%7C1549839468424%3B; IR_8154=1547247468480%7C0%7C1547247402943%7C308SnG1k%3Ax6NRl41WlS2czhgUkgycW2FszrdRM0; IR_PI=1538887702311.ypfh9yayldq%7C1547333868480; s_sess=%20stsh%3D%3B%20s_pv_pName%3Dgift%2520card%253Ebalance%2520check%3B%20s_pv_pType%3Dgift%2520card%3B%20s_pv_cmpgn%3D%3B%20s_cc%3Dtrue%3B%20s_sq%3Dhomedepotprod%25252Chomedepotglobaldev%253D%252526c.%252526a.%252526activitymap.%252526page%25253Dgift%25252520card%2525253Ebalance%25252520check%252526link%25253DCheck%25252520Balance%252526region%25253Dapp%252526pageIDType%25253D1%252526.activitymap%252526.a%252526.c%252526pid%25253Dgift%25252520card%2525253Ebalance%25252520check%252526pidt%25253D1%252526oid%25253Dfunctionpr%25252528%25252529%2525257B%2525257D%252526oidt%25253D2%252526ot%25253DSUBMIT%3B", "cache-control": "no-cache", } ) payload = { "GiftCardsRequest": { "cardNumber": kwargs["card_number"], "pinNumber": kwargs["pin"], "reCaptcha": captcha_resp["solution"]["gRecaptchaResponse"], } } logger.info(f"Fetching balance from API") try: resp = session.post(self.api_endpoint, json=payload, timeout=5) except requests.exceptions.RequestException as e: raise RuntimeError(f"Error on API post: {e}") if resp.status_code != 200: raise RuntimeError( f"Failed to get valid response from API (status code {resp.status_code})" ) print(resp.text) quit()
def check_balance(self, **kwargs): if self.validate(kwargs): logger.info("Checking balance for card: {}, exp {}/{}".format( kwargs["card_number"], kwargs["exp_month"], kwargs["exp_year"])) return self.scrape({ "ctl00$ctl00$FullContent$MainContent$tbNumber": kwargs["card_number"], "ctl00$ctl00$FullContent$MainContent$tbExpDate": kwargs["exp_month"] + kwargs["exp_year"], "ctl00$ctl00$FullContent$MainContent$tbCid": kwargs["cvv"], })
def scrape(self, cards_chunk): session = requests.Session() session.headers.update({ "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en;q=0.9", }) # Get balance page in session to retrieve cookies session.get(self.website_url) # Assemble a string for JSON request from cards chunk taken cards_str = "" for i, card in enumerate(cards_chunk): if i > 0: cards_str += "," cards_str += card["card_number"] + ":" + card["pin"] payload = { "gift_card_numbers": cards_str, "action": "checkCertificateBalance", "country": "US", "lang_locale": "en_US", } session.headers.update( {"content-type": "application/x-www-form-urlencoded"}) logger.info(f"Fetching balance from API") try: resp = session.post(self.api_endpoint, data=payload, timeout=5) except requests.exceptions.RequestException as e: raise RuntimeError(f"Error on API post: {e}") if resp.status_code != 200: raise RuntimeError( f"Failed to get valid response from API (status code {resp.status_code})" ) print(resp.text) quit()
def scrape(self, **kwargs): logger.info("Solving reCAPTCHA (~30s)") captcha_solver = CaptchaSolver(api_key=config.ANTI_CAPTCHA_KEY) # Site key obtained from: https://www.homedepot.com/mycheckout/assets/react/giftcard.bundle.1.2302.0.js captcha_resp = captcha_solver.solve_recaptcha( self.website_url, "6LfEHBkTAAAAAHX6YgeUw9x1Sutr7EzhMdpbIfWJ") if captcha_resp["errorId"] != 0: raise RuntimeError( f"Unable to solve reCAPTCHA ({captcha_resp['errorDescription']})" ) payload = { "GiftCardsRequest": { "cardNumber": kwargs["card_number"], "pinNumber": kwargs["pin"], "reCaptcha": captcha_resp["solution"]["gRecaptchaResponse"], } } logger.info("Fetching balance from API") try: resp = requests.post(self.api_endpoint, json=payload, headers=HEADERS) if resp.status_code != 200: raise RuntimeError( f"Invalid API response (status code {resp.status_code})") result = deep_get(resp.json(), "giftCards.giftCard") if result is None: raise RuntimeError( f"Invalid API response: unable to find giftCard key in JSON response" ) err_code = deep_get(result, "errorCode") if err_code: err_desc = deep_get(result, "description") raise RuntimeError( f"Failed to retrieve balance from API: {err_desc} ({err_code})" ) initial_balance = deep_get(result, "originalAmount") avail_balance = deep_get(result, "availableAmount") logger.info(f"Success! Card balance: {avail_balance}") return { "initial_balance": initial_balance, "available_balance": avail_balance, } except requests.exceptions.RequestException as e: raise RuntimeError(f"Error on API post: {e}") except JSONDecodeError: raise RuntimeError("Failed to parse API response as JSON")
def scrape(self, fields): # Open Selenium browser browser = webdriver.Chrome() logger.info("Fetching balance check page") browser.get(self.website_url) logger.info("Filling balance check form") for field, val in fields.items(): try: browser.find_element_by_id(field).send_keys(val) time.sleep(1) except NoSuchElementException: browser.close() raise RuntimeError(f"Unable to find '{field}' field on page") # Click submit button browser.find_element_by_id("brandLoginForm_button").click() # Wait for page to load try: WebDriverWait(browser, 3).until( EC.presence_of_element_located((By.ID, "Avlbal"))) except TimeoutException: browser.close() raise RuntimeError("Balance page took too long to load") logger.info("Obtaining card information") try: avail_balance = browser.find_element_by_id("Avlbal").text except NoSuchElementException: browser.close() raise RuntimeError("Could not find available card balance") try: initial_balance = (browser.find_element_by_class_name( "rightSide").find_element_by_tag_name("span").text) except NoSuchElementException: browser.close() raise RuntimeError("Could not find initial card balance") browser.close() logger.info(f"Success! Card balance: {avail_balance}") return { "initial_balance": initial_balance, "available_balance": avail_balance }
async def scrape(self, fields): browser = await pyppeteer.launch(handleSIGINT=False, handleSIGTERM=False, handleSIGHUP=False) page = await browser.newPage() logger.info("Fetching balance check page") await page.goto(self.website_url) logger.info("Filling balance check form") await page.type("#Card_Number", fields["card_number"]) await page.type("#Card_Pin", fields["pin"]) logger.info("Requesting balance") await page.click("#CheckBalance button") await page.waitForSelector(".fetch_balance_value", {"timeout": 10000}) avail_balance = await page.querySelectorEval( ".fetch_balance_value", "(node => node.innerText)") logger.info("Success! Card balance: {}".format(avail_balance)) return {"available_balance": avail_balance}
def scrape(self, **kwargs): session = requests.Session() session.headers.update({ "User-Agent": self.ua.random, # fake UA "accept": "application/json", "origin": "https://www.bestbuy.com", "content-type": "application/json", "referer": "https://www.bestbuy.com/digitallibrary/giftcard", "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en;q=0.9", "cache-control": "no-cache", }) payload = f'{{"cardNumber":"{kwargs["card_number"]}","pin":"{kwargs["pin"]}"}}' logger.info(f"Fetching balance from API") try: resp = session.post(self.website_url, data=payload, timeout=5) except requests.exceptions.RequestException as e: raise RuntimeError(f"Error on API post: {e}") if resp.status_code != 200: raise RuntimeError( f"Failed to get valid response from API (status code {resp.status_code})" ) # TODO: Sometimes does not output balance or throw error, not sure why. Happens with IP temp block try: avail_balance = resp.json()["balance"] except: raise RuntimeError("Could not parse balance from JSON response") logger.info(f"Success! Card balance: {avail_balance}") # TODO: figure out cleaner way to do this, feature in main? self.num_runs += 1 # Not sure exactly what sweet spot is but IP blocks at 10 min, 15 min seems good seconds = 3 if self.num_runs % 9 else 60 * 15 + 5 logger.info( f"Ran {self.num_runs} times. Sleeping {seconds} seconds before trying next..." ) time.sleep(seconds) return {"balance": avail_balance}
def scrape(self, fields): # Open Selenium browser browser = webdriver.Chrome() browser.set_window_size(600, 800) logger.info("Fetching balance check page") browser.get(self.website_url) try: form = browser.find_elements_by_tag_name("form")[1] except (NoSuchElementException, IndexError): raise RuntimeError("Unable to find login form on page") logger.info("Filling login form 1/2") try: form.find_element_by_name("accountNumber").send_keys( fields["accountNumber"]) time.sleep(1) except NoSuchElementException: browser.close() raise RuntimeError(f"Unable to find 'accountNumber' field on page") # Click continue button form.find_element_by_css_selector("input[type='submit']").click() # Wait for page to load try: WebDriverWait(browser, 3).until( EC.presence_of_element_located((By.ID, "login-form"))) except TimeoutException: browser.close() raise RuntimeError("Login page took too long to load") try: form = browser.find_element_by_id("login-form") except NoSuchElementException: browser.close() raise RuntimeError("Unable to find login form on page") logger.info("Solving CAPTCHA (~10s)") # Extract CAPTCHA image from page captcha_b64 = get_image_b64_by_id(browser, "captchaImg") captcha_solver = CaptchaSolver(api_key=config.ANTI_CAPTCHA_KEY) captcha = captcha_solver.solve_image_b64(captcha_b64) if captcha["errorId"] != 0: browser.close() raise RuntimeError("Unable to solve CAPTCHA ({})".format( captcha["errorDescription"])) logger.info("Filling login form 2/2") try: form.find_element_by_name("cv2").send_keys(fields["cv2"]) time.sleep(1) except NoSuchElementException: browser.close() raise RuntimeError("Unable to find 'cv2' field on page") try: form.find_element_by_id( "_MultiStageFSVpasswordloginresponsive_WAR_cardportalresponsive_captchaText" ).send_keys(captcha["solution"]["text"]) time.sleep(1) except NoSuchElementException: browser.close() raise RuntimeError("Unable to find CAPTCHA field on page") # Click continue button form.find_element_by_css_selector("input[type='submit']").click() # Wait for page to load try: WebDriverWait(browser, 3).until( EC.presence_of_element_located((By.ID, "cardBalanceInfo"))) except TimeoutException: browser.close() raise RuntimeError("Balance page took too long to load") logger.info("Obtaining card information") try: avail_balance = browser.find_element_by_class_name( "cardBalanceText").text except NoSuchElementException: browser.close() raise RuntimeError("Could not find available card balance") browser.close() logger.info(f"Success! Card balance: {avail_balance}") return {"initial_balance": None, "available_balance": avail_balance}
def scrape(self, fields): session = requests.Session() session.headers.update({"User-Agent": config.USER_AGENT}) logger.info("Fetching balance check page") resp = session.get(self.website_url) if resp.status_code != 200: raise RuntimeError( f"Failed to GET website (status code {resp.status_code}") page_html = BeautifulSoup(resp.content, features="html.parser") form = page_html.find("form") if not form: raise RuntimeError("Unable to find balance check form") endpoint = f"{self.website_url}{form['action']}" # These fields are present on GS balance check page, does not work without including them fields['__EVENTTARGET'] = '' fields['__EVENTARGUMENT'] = '' fields['__LASTFOCUS'] = '' fields['__VIEWSTATE'] = page_html.find("input", id='__VIEWSTATE')['value'] fields['__VIEWSTATEGENERATOR'] = page_html.find( "input", id='__VIEWSTATEGENERATOR')['value'] recaptcha_field = page_html.find("div", class_="g-recaptcha") if not recaptcha_field: raise RuntimeError("Unable to find reCAPTCHA") site_key = recaptcha_field["data-sitekey"] logger.info("Solving reCAPTCHA (~30s)") captcha_resp = captcha_solver.solve_recaptcha(self.website_url, site_key) if captcha_resp["errorId"] != 0: raise RuntimeError( f"Unable to solve reCAPTCHA ({captcha_resp['errorDescription']})" ) fields["g-recaptcha-response"] = captcha_resp["solution"][ "gRecaptchaResponse"] logger.info("Fetching card balance") session.headers.update({ # Not necessary for this merchant }) form_resp = session.post(endpoint, data=fields) if form_resp.status_code != 200: raise RuntimeError( f"Failed to retrieve card balance (status code {form_resp.status_code})" ) balance_html = BeautifulSoup(form_resp.content, features="html.parser") try: avail_balance = balance_html.find("span", class_='balancePrice').text except: # GS prunes old depleted cards from its system and throws an 'invalid' error # Set these as -1 to notate they gave invalid error but are likely actually zero balance if balance_html.find( text='The Gift Card number entered is invalid.'): avail_balance = '-1' elif balance_html.find( 'span', id= 'BaseContentPlaceHolder_mainContentPlaceHolder_recaptchaMessage' ): # Message on screen: "The code you entered is invalid." # CAPTCHA answer invalid, retry raise RuntimeError( 'Invalid CAPTCHA answer supplied! Trying again...') else: raise RuntimeError( 'Could not find balance on retrieved page for unknown reason' ) logger.info(f"Success! Card balance: {avail_balance}") return ({"balance": avail_balance})
def check_balance(self, **kwargs): if self.validate(kwargs): logger.info(f"Checking balance for card: {kwargs['card_number']}") return self.scrape(number=kwargs["card_number"], pin=kwargs["pin"])
def main(): providers_help = "\n".join( [" - {}".format(p_name) for p_name in providers.keys()]) parser = ArgumentParser( formatter_class=RawTextHelpFormatter, description=f"""Check gift card balances for a variety of providers. Supported providers: {providers_help} Requires an Anti-CAPTCHA API key for providers with CAPTCHAs. Get one here: https://anti-captcha.com Configure your key by setting the ANTI_CAPTCHA_KEY environment variable. Your INPUT_CSV should be formatted as follows: - A header row is required - Each column should contain a parameter required by the specified provider Example (for the 'blackhawk' provider): ------------------------------------------------- | card_number | exp_month | exp_year | cvv | |------------------|-----------|----------|-----| | 4111111111111111 | 12 | 24 | 999 | ------------------------------------------------- If you find this tool useful, consider buying a coffee for the author: https://stevenmirabito.com/kudos""", ) parser.add_argument("-v", "--version", action="version", version=f"%(prog)s {version.__version__}") parser.add_argument( "provider", metavar="PROVIDER", type=str.lower, help="Name of balance check provider", ) parser.add_argument("input", metavar="INPUT_CSV", type=str, help="Path to input CSV") parser.add_argument( "--output", "-o", metavar="OUTPUT_CSV", type=str, help=("Path to output CSV (optional; default:" "add/overwrite\nbalance columns on input CSV)"), ) args = parser.parse_args() in_filename = path.abspath(args.input) out_filename = in_filename if not args.output else path.abspath( args.output) if args.provider not in providers: logger.fatal(f"Unknown provider: '{args.provider}'") sys.exit(1) provider = providers[args.provider] max_workers = (provider.max_workers if hasattr(provider, "max_workers") else config.MAX_WORKERS) provider_allows_chunks = True if hasattr(provider, "max_simultaneous") else False futures = {} results = [] retries = {} with ThreadPoolExecutor(max_workers=max_workers) as executor: try: with open(in_filename, newline="") as input_csv: reader = csv.DictReader(input_csv) _chunk = [] for i, card_data in enumerate(reader): # Add the card details to the result results.append(card_data) # Some balance checkers accept multiple cards so prepare to send chunks # !! NEEDS WORK, UNFINISHED if provider_allows_chunks: _chunk.append(card_data) if ( i + 1 ) % provider.max_simultaneous: # If end of chunk, send to schedule... # Schedule balance check future = executor.submit(provider.check_balance, _chunk) futures[future] = i _chunk = [] # Clear chunk at end else: future = executor.submit(provider.check_balance, **card_data) futures[future] = i # Done reading input file and scheduling logger.info(f"Read {len(results)} cards from file '{in_filename}'") except (OSError, IOError) as err: logger.fatal(f"Unable to open input file '{in_filename}': {err}") sys.exit(1) except Exception as e: logger.fatal(f"Unexpected error: {e}") sys.exit(1) # While there are still tasks queued, jump back in (handles retries) while futures: # Update progress bar as tasks complete for future in tqdm(as_completed(futures), total=len(futures), leave=False): idx = futures.pop(future) try: balance_info = future.result() except Exception as e: # Log the first column value as an ID (usually card number) card_id = next(iter(results[idx].values())) # Attempt to schedule retry if idx in retries: retries[idx] += 1 if retries[idx] > config.RETRY_TIMES: # Out of retries, permanent failure logger.error( f"Failed to balance check {card_id} (out of retries). Last error: {e}" ) else: executor.submit(logger.error, "error occurred", exc_info=sys.exc_info()) retries[idx] = 1 logger.warning( "RETRY {}/{}: Failed to balance check {}, will retry later. Error: {}" .format(retries[idx], config.RETRY_TIMES, card_id, e)) future = executor.submit(provider.check_balance, **results[idx]) futures[future] = idx else: # Successful balance(s) returned # !! NEEDS WORK, UNFINISHED if provider_allows_chunks: # List of balances from chunk of cards returned for i, balance_info in enumerate(balances_info): results[idx] = dict(results[idx], **balance_info) # If not on last cards balance info... if len(balance_info) - 1 != i: idx += 1 else: # Single balance returned results[idx] = dict(results[idx], **balance_info) try: with open(out_filename, "w", newline="") as output_csv: logger.info(f"Writing CSV output to {out_filename}...") fieldnames = results[0].keys() writer = csv.DictWriter(output_csv, fieldnames=fieldnames) writer.writeheader() for row in results: writer.writerow(row) logger.info(f"Output written to: {out_filename}") except (OSError, IOError) as err: logger.fatal(f"Unable to open output file '{in_filename}': {err}") sys.exit(1) except Exception as e: logger.fatal(f"Unexpected error: {e}") sys.exit(1)
def scrape(self, fields): session = requests.Session() session.headers.update({"User-Agent": config.USER_AGENT}) logger.info("Fetching balance check page") resp = session.get(self.website_url) if resp.status_code != 200: logger.critical( f"Failed to GET Simon website (status code {resp.status_code})" ) sys.exit(1) page_html = BeautifulSoup(resp.content, features="html.parser") recaptcha_field = page_html.find("div", class_="g-recaptcha") if not recaptcha_field: logger.critical("Unable to find reCAPTCHA") sys.exit(1) site_key = recaptcha_field["data-sitekey"] logger.info("Solving reCAPTCHA (~30s)") captcha = captcha_solver.solve_recaptcha(self.website_url, site_key) if captcha["errorId"] != 0: logger.critical( f"Unable to solve reCAPTCHA ({captcha['errorDescription']})") sys.exit(1) # These fields are present on balance check page, request blocked if not included fields["__EVENTTARGET"] = "" fields["__EVENTARGUMENT"] = "" fields["__LASTFOCUS"] = "" fields["__VIEWSTATE"] = page_html.find("input", id="__VIEWSTATE")["value"] fields["__VIEWSTATEGENERATOR"] = page_html.find( "input", id="__VIEWSTATEGENERATOR")["value"] fields["__VIEWSTATEENCRYPTED"] = "" fields["__EVENTVALIDATION"] = page_html.find( "input", id="__EVENTVALIDATION")["value"] fields["ctl00$ctl00$header1$EmailLogin"] = "" fields["ctl00$ctl00$header1$PasswordLogin"] = "" fields[ "ctl00$ctl00$FullContent$MainContent$checkBalanceSubmit"] = "CHECK YOUR BALANCE" fields["g-recaptcha-response"] = captcha["solution"][ "gRecaptchaResponse"] session.headers.update({ "User-Agent": config.USER_AGENT, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept-Encoding": "gzip, deflate, br", "Accept-Language": "en-US,en;q=0.9", "Cache-Control": "no-cache", "Connection": "keep-alive", "Content-Type": "application/x-www-form-urlencoded", "Referer": self.website_url, "origin": "https://www.simon.com", }) logger.info("Fetching card balance") form_resp = session.post(self.website_url, data=fields) if form_resp.status_code != 200: logger.critical( f"Failed to retrieve card balance (status code {form_resp.status_code})" ) sys.exit(1) balance_html = BeautifulSoup(form_resp.content, features="html.parser") if balance_html.find("label", text="CAPTCHA: Please validate"): raise RuntimeError("Invalid CAPTCHA answer supplied.") try: avail_balance = balance_html.find( id="ctl00_ctl00_FullContent_MainContent_lblBalance").text initial_balance = balance_html.find_all( "td", _class="tblinfo") # [-1].text.strip() print(initial_balance) except: print("DUMP:", resp.text) raise RuntimeError("Could not find balance on page") logger.info(f"Success! Card balance: {avail_balance}") return { "initial_balance": initial_balance, "available_balance": avail_balance }
def main(): providers_help = "\n".join( [" - {}".format(p_name) for p_name in providers.keys()]) parser = ArgumentParser( formatter_class=RawTextHelpFormatter, description="""Check gift card balances for a variety of providers. Supported providers: {} Requires an Anti-CAPTCHA API key for providers with CAPTCHAs. Get one here: https://anti-captcha.com Configure your key by setting the ANTI_CAPTCHA_KEY environment variable. Your INPUT_CSV should be formatted as follows: - A header row is required - Each column should contain a parameter required by the specified provider Example (for the 'blackhawk' provider): ------------------------------------------------- | card_number | exp_month | exp_year | cvv | |------------------|-----------|----------|-----| | 4111111111111111 | 12 | 24 | 999 | ------------------------------------------------- If you find this tool useful, consider buying a coffee for the author: https://stevenmirabito.com/kudos""".format(providers_help), ) parser.add_argument( "-v", "--version", action="version", version="%(prog)s {}".format(version.__version__), ) parser.add_argument( "provider", metavar="PROVIDER", type=str.lower, help="Name of balance check provider", ) parser.add_argument("input", metavar="INPUT_CSV", type=str, help="Path to input CSV") parser.add_argument( "--output", "-o", metavar="OUTPUT_CSV", type=str, help=("Path to output CSV (optional; default:" "add/overwrite\nbalance columns on input CSV)"), ) args = parser.parse_args() in_filename = path.abspath(args.input) out_filename = in_filename if args.output: # Separate output path specified out_filename = path.abspath(args.output) if args.provider not in providers: logger.fatal("Unknown provider: '{}'".format(args.provider)) sys.exit(1) provider = providers[args.provider] futures = {} results = [] retries = {} with ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: try: with open(in_filename, newline="") as input_csv: reader = csv.DictReader(input_csv) for row in reader: # Add the card details to the result results.append(row) idx = len(results) - 1 # Schedule balance check future = executor.submit(provider.check_balance, **row) futures[future] = idx except (OSError, IOError) as err: logger.fatal("Unable to open input file '{}': {}".format( in_filename, err)) sys.exit(1) except Exception as e: logger.fatal("Unexpected error: {}".format(e)) sys.exit(1) # While there are still tasks queued, jump back in (handles retries) while futures: # Update progress bar as tasks complete for future in tqdm(as_completed(futures), total=len(futures), leave=False): idx = futures.pop(future) try: balance_info = future.result() except Exception as e: # Log the first column value as an ID (usually card number) card_id = next(iter(results[idx].values())) # Attempt to schedule retry if idx in retries: retries[idx] += 1 if retries[idx] > config.RETRY_TIMES: # Out of retries, permanent failure logger.error( "Failed to balance check {} (out of retries). Last error: {}" .format(card_id, e)) else: retries[idx] = 1 logger.warning( "RETRY {}/{}: Failed to balance check {}, retrying. Error: {}" .format(retries[idx], config.RETRY_TIMES, card_id, e)) future = executor.submit(provider.check_balance, **results[idx]) futures[future] = idx else: # Combine original card details with balance information results[idx] = dict(results[idx], **balance_info) try: with open(out_filename, "w", newline="") as output_csv: logger.info("Writing output CSV...") fieldnames = results[0].keys() writer = csv.DictWriter(output_csv, fieldnames=fieldnames) writer.writeheader() for row in results: writer.writerow(row) logger.info("Output written to: {}".format(out_filename)) except (OSError, IOError) as err: logger.fatal("Unable to open output file '{}': {}".format( in_filename, err)) sys.exit(1) except Exception as e: logger.fatal("Unexpected error: {}".format(e)) sys.exit(1)
def scrape(self, **kwargs): cookies = CookieJar() opener = request.build_opener(request.HTTPCookieProcessor(cookies)) logger.info("Fetching balance check page") # Use urllib directly as requests gets blocked req = request.Request(self.website_url, headers=HEADERS) resp = opener.open(req) if resp.status != 200: raise RuntimeError( f"Failed to get GameStop website (status code {resp.status})") page_html = BeautifulSoup(resp.read(), features="html.parser") recaptcha_el = page_html.find("div", {"data-sitekey": True}) if not recaptcha_el: raise RuntimeError("Unable to find reCAPTCHA on page") csrf_el = page_html.find("input", {"name": "csrf_token"}) if not csrf_el: raise RuntimeError("Unable to find CSRF on page") site_key = recaptcha_el["data-sitekey"] csrf_token = csrf_el["value"] logger.info("Solving reCAPTCHA (~30s)") captcha_solver = CaptchaSolver(api_key=config.ANTI_CAPTCHA_KEY) captcha_resp = captcha_solver.solve_recaptcha(self.website_url, site_key) if captcha_resp["errorId"] != 0: raise RuntimeError( f"Unable to solve reCAPTCHA ({captcha_resp['errorDescription']})" ) payload = { "dwfrm_giftCard_balance_accountNumber": kwargs["card_number"], "dwfrm_giftCard_balance_pinNumber": kwargs["pin"], "g-recaptcha-response": captcha_resp["solution"]["gRecaptchaResponse"], "csrf_token": csrf_token, } logger.info("Fetching balance from API") try: req = request.Request(self.api_endpoint, headers=HEADERS) data = urlencode(payload).encode("utf-8") req.add_header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") req.add_header("Content-Length", len(data)) resp = opener.open(req, data) if resp.status != 200: raise RuntimeError( f"Invalid API response (status code {resp.status})") result = json.loads(resp.read().decode( resp.info().get_param("charset") or "utf-8")) errors = deep_get(result, "error") if errors: err = errors[0] raise RuntimeError( f"Failed to retrieve balance from API: {err}") balance = deep_get(result, "balance") if balance is None: raise RuntimeError( f"Invalid API response: unable to find required key in JSON response" ) logger.info(f"Success! Card balance: ${balance}") return { "balance": f"${balance}", } except json.JSONDecodeError: raise RuntimeError("Failed to parse API response as JSON")
def scrape(self, fields): session = requests.Session() session.headers.update({ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept-Encoding": "gzip, deflate, br", "Accept-Language": "en-US,en;q=0.9", "Connection": "keep-alive", }) logger.info("Fetching balance check page") resp = session.get(self.website_url) if resp.status_code != 200: logger.critical( f"Failed to GET OneVanilla website (status code {resp.status_code})" ) sys.exit(1) print(resp.text) page_html = BeautifulSoup(resp.content, features="html.parser") action = page_html.find("form")["action"] # brandLoginForm fields["csrfToken"] = page_html.find("input", name="csrfToken")["value"] session.headers.update({ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept-Encoding": "gzip, deflate, br", "Accept-Language": "en-US,en;q=0.9", "Cache-Control": "no-cache", "Connection": "keep-alive", "Content-Type": "application/x-www-form-urlencoded", "Referer": self.website_url, "origin": "https://www.simon.com", }) logger.info("Fetching card balance") form_resp = session.post(self.website_url + action[2:], data=fields) if form_resp.status_code != 200: logger.critical( f"Failed to retrieve card balance (status code {form_resp.status_code})" ) sys.exit(1) balance_html = BeautifulSoup(form_resp.content, features="html.parser") try: avail_balance = balance_html.find("li", id="Avlbal").text.strip() # initial_balance = balance_html.find("li", text="Original Value:") #[-1].text.strip() initial_balance = "1" except: print("DUMP:", resp.text) raise RuntimeError("Could not find balance on page") logger.info(f"Success! Card balance: {avail_balance}") return { "initial_balance": initial_balance, "available_balance": avail_balance }
def scrape(self, fields): session = requests.Session() session.headers.update({"User-Agent": config.USER_AGENT}) fields["X-Requested-With"] = "XMLHttpRequest" logger.info("Fetching balance check page") resp = session.get(self.website_url) if resp.status_code != 200: logger.critical( "Failed to GET Blackhawk website (status code {})".format( resp.status_code)) sys.exit(1) page_html = BeautifulSoup(resp.content, features="html.parser") transactions = page_html.find(id="CheckBalanceTransactions") form = transactions.find("form") if not form: logger.critical("Unable to find balance check form") sys.exit(1) endpoint = "{}{}".format(self.website_url, form["action"]) token_field = transactions.find( "input", attrs={"name": "__RequestVerificationToken"}) if not token_field: logger.critical("Failed to retrieve verification token") sys.exit(1) fields["__RequestVerificationToken"] = token_field["value"] recaptcha_field = transactions.find("div", class_="g-recaptcha") if not recaptcha_field: logger.critical("Unable to find reCAPTCHA") sys.exit(1) site_key = recaptcha_field["data-sitekey"] logger.info("Solving reCAPTCHA (~30s)") captcha = captcha_solver.solve_recaptcha(self.website_url, site_key) if captcha["errorId"] != 0: logger.critical("Unable to solve reCAPTCHA ({})".format( captcha["errorDescription"])) sys.exit(1) fields["g-recaptcha-response"] = captcha["solution"][ "gRecaptchaResponse"] logger.info("Fetching card balance") session.headers.update({ "Accept": "*/*", "Accept-Encoding": "gzip, deflate, br", "Accept-Language": "en-US,en;q=0.5", "Cache-Control": "no-cache", "Connection": "keep-alive", "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", "Pragma": "no-cache", "Referer": "https://mygift.giftcardmall.com/", "X-Requested-With": "XMLHttpRequest", }) form_resp = session.post(endpoint, data=fields) if form_resp.status_code != 200: logger.critical( "Failed to retrieve card balance (status code {})".format( form_resp.status_code)) sys.exit(1) balance_html = BeautifulSoup(form_resp.content, features="html.parser") avail_balance = (balance_html.find( "div", text="Available Balance").parent.find("div", class_="value").text) initial_balance = (balance_html.find( "div", text="Initial Balance").parent.find("div", class_="value").text) logger.info("Success! Card balance: {}".format(avail_balance)) return { "initial_balance": initial_balance, "available_balance": avail_balance }
def scrape(self, fields): session = requests.Session() session.headers.update({"User-Agent": config.USER_AGENT}) logger.info("Fetching balance check page") resp = session.get(self.website_url) if resp.status_code != 200: raise RuntimeError( "Failed to GET Spafinder website (status code {})".format( resp.status_code)) page_html = BeautifulSoup(resp.content, features="html.parser") inquiry = page_html.find(id="balance-inquiry") form = inquiry.find("form") if not form: raise RuntimeError("Unable to find balance check form") endpoint = "{}{}".format(self.base_url, form["action"]) # Page has bad HTML, need to search from top level recaptcha_field = page_html.find("div", class_="g-recaptcha") if not recaptcha_field: raise RuntimeError("Unable to find reCAPTCHA") site_key = recaptcha_field["data-sitekey"] logger.info("Solving reCAPTCHA (~30s)") captcha_solver = CaptchaSolver(api_key=config.ANTI_CAPTCHA_KEY) captcha_resp = captcha_solver.solve_recaptcha(self.website_url, site_key) if captcha_resp["errorId"] != 0: raise RuntimeError("Unable to solve reCAPTCHA ({})".format( captcha_resp["errorDescription"])) fields["g-recaptcha-response"] = captcha_resp["solution"][ "gRecaptchaResponse"] logger.info("Fetching card balance") form_resp = session.post(endpoint, data=fields) if form_resp.status_code != 200: raise RuntimeError( "Failed to retrieve card balance (status code {})".format( form_resp.status_code)) resp_html = BeautifulSoup(form_resp.content, features="html.parser") error_html = resp_html.find("div", class_="alert-danger") if error_html: raise RuntimeError("Got error while checking balance: {}".format( error_html.text)) balance_container = resp_html.find("div", class_="alert-success") if not balance_container: raise RuntimeError("Unable to find balance container") balance_match = re.search("(\d{1,2}\.\d{2})", balance_container.get_text()) if not balance_match: raise RuntimeError("Unable to find balance text") balance = "${}".format(balance_match.group(0)) logger.info("Success! Card balance: {}".format(balance)) return {"balance": balance}