def scrape(self, **card):
session = requests.Session()
session.headers.update({
"user-agent": config.USER_AGENT,
"accept": "*/*",
"accept-encoding": "gzip, deflate, br",
"accept-language": "en-US,en;q=0.9",
})
params = {"giftCardNumber": card["number"], "pin": card["pin"]}
logger.info(f"Fetching balance")
resp = session.get(self.website_url, params=params)
if resp.status_code != 200:
raise RuntimeError(
f"Failed to retrieve card balance (status code {resp.status_code})"
)
# Tried to use BS4 but it refused to work, I think HTML returned was too messy/non-compliant
page_parsed = lxml.html.fromstring(resp.text)
try:
avail_balance = page_parsed.xpath(
"//div[@class='cardPoints']/div")[0].text
except:
# error on screen
error_text = page_parsed.xpath(
"//div[contains(@class,'error')]")[0].text
raise RuntimeError(error_text)
logger.info(f"Success! Card balance: {avail_balance}")
return {"balance": avail_balance}
def check_balance(self, cards_chunk):
for card in cards_chunk:
if not self.validate(card):
raise RuntimeError(
f'Card format of {card["card_number"]} failed validation')
logger.info(f"Checking balance for cards in chunk")
return self.scrape(cards_chunk)
def check_balance(self, **kwargs):
if self.validate(kwargs):
logger.info("Checking balance for card: {}".format(
kwargs["card_number"]))
return self.scrape({
"accountNumber": kwargs["card_number"],
"cv2": kwargs["cvv"]
})
def check_balance(self, **kwargs):
if self.validate(kwargs):
logger.info(f"Checking balance for card: {kwargs['card_number']}")
form_inputs = {
"ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$CardNumberTextBox":
kwargs["card_number"],
"ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$PinTextBox":
kwargs["pin"],
}
return self.scrape(form_inputs)
def check_balance(self, **kwargs):
if self.validate(kwargs):
logger.info("Checking balance for card: {}, exp {}/{}".format(
kwargs["card_number"], kwargs["exp_month"],
kwargs["exp_year"]))
return self.scrape({
"CardNumber": kwargs["card_number"],
"ExpirationDateMonth": kwargs["exp_month"],
"ExpirationDateYear": kwargs["exp_year"],
"SecurityCode": kwargs["cvv"],
})
def check_balance(self, **kwargs):
if self.validate(kwargs):
logger.info("Checking balance for card: {}, exp {}/{}".format(
kwargs["card_number"], kwargs["exp_month"],
kwargs["exp_year"]))
return self.scrape({
"cardNumber": kwargs["card_number"],
"expMonth": kwargs["exp_month"],
"expYear": kwargs["exp_year"],
"cvv": kwargs["cvv"],
})
def check_balance(self, **kwargs):
if self.validate(kwargs):
logger.info("Checking balance for card: {}".format(
kwargs["card_number"]))
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
return loop.run_until_complete(
self.scrape({
"card_number": kwargs["card_number"],
"pin": kwargs["pin"],
}))
def check_balance(self, **kwargs):
if self.validate(kwargs):
logger.info("Checking balance for card: {}, exp {}/{}".format(
kwargs["card_number"], kwargs["exp_month"],
kwargs["exp_year"]))
return self.scrape({
"number-1": kwargs["card_number"],
"valid-mm": str(int(kwargs["exp_month"])
), # Lazy way to strip '0' prefix, if present
"valid-yy": "20{}".format(kwargs["exp_year"]),
"pin": kwargs["cvv"],
})
def scrape(self, **kwargs):
# Site key gathered from https://secure2.homedepot.com/mycheckout/assets/react/giftcard.bundle.js?v=v1.2040.2
# 6LfEHBkTAAAAAHX6YgeUw9x1Sutr7EzhMdpbIfWJ and 6Le3GRkTAAAAAPpXON0jcJCLrYZnm-ZqyLhbCLbX
captcha_resp = captcha_solver.solve_recaptcha(
self.website_url, "6LfEHBkTAAAAAHX6YgeUw9x1Sutr7EzhMdpbIfWJ"
)
if captcha_resp["errorId"] != 0:
raise RuntimeError(
f"Unable to solve reCAPTCHA ({captcha_resp['errorDescription']})"
)
# Begin API request
session = requests.Session()
# Get balance check page to grab cookies
session.get(website_url)
session.headers.update(
{
"origin": "https://secure2.homedepot.com",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
"content-type": "application/json",
"accept": "application/json, text/plain, */*",
"referer": "https://secure2.homedepot.com/mycheckout/giftcard",
"accept-encoding": "gzip, deflate, br",
"accept-language": "en-US,en;q=0.9",
#'cookie': "HD_DC=origin; check=true; _abck=3D7AEA27759869A787304AD8C0B93D1C17C532E781200000EB2DB15ACADB6E34~0~Ie8mODWyYhnd9Jka+w+AC39XvfEAlhz78nuYgOCT8so=~-1~-1; AMCVS_F6421253512D2C100A490D45%40AdobeOrg=1; thda.u=5f480787-7953-8e2e-14c3-0b96cfd48040; thda.s=054d35ff-11c7-6b16-75cd-bbcd2d48c17b; RES_TRACKINGID=85286815566469932; ftr_ncd=6; thda.m=81084504873679911179154344636083105898; LPVID=Q5OGUwNTliM2EzY2NmM2Yw; cto_lwid=2dc94469-69f2-49af-886c-69251e0ee8da; THD_FORCE_LOC=1; THD_MCC_ID=6f1d76fc-8b8e-45be-97ea-9c061f79f851; cart_activity=8777201b-f4c7-4ccc-8e3e-428c62d3c730; WORKFLOW=LOC_HISTORY_BY_IP; THD_INTERNAL=0; og_session_id=379dc8f09a4311e7806bbc764e106cf4.731928.1538275068; IR_gbd=homedepot.com; ecrSessionId=6ED9C65DD7DE929C82CAEB9D58E38D71; THD_USER=\"eyJzdm9jQ3VzdG9tZXJBY2NvdW50SWQiOiIwM0VEMjFBQ0JDNTgzNTkwMFMiLCJsb2dvbklkIjoiaG9tZWRlcG90QGJpa2VtYW5kYW4uY29tIiwidXNlcklkIjoiMDUxM0UyMTkyOTIxRTY5MTBVIiwiY3VzdG9tZXJUeXBlIjoiQjJDIn0=\"; THD_USER_SESSION=AQIC5wM2LY4SfcxJD0XieCwTVsVkLiR7a2oDQMG6d90y3To.*AAJTSQACMDIAAlNLABM2MzI5MDk2NzEyMTU1OTY3NjQ4AAJTMQACMDY.*; THD_CACHE_NAV_PERSIST=\"\"; THD_CACHE_NAV_SESSION=C20%7E0_%7EC20_EXP%7E_%7EC22%7E641_%7EC22_EXP%7E_%7EC26%7EP_REP_PRC_MODE%7C0_%7EC26_EXP%7E; ak_bmsc=C6D4C7801E4881DEBB1E03DF88F8F1CF17C532D4FC2700009D15395C49757425~plfpoGCv+yaNOcuoWld8wCQ3wGC2Yh+Dl/XvPv5aTAd9s7O3guYCefrdJKkL4e+oVw8WlPV3onzF+qQyqCQ0fQrP9dWjPPDzi2FDN8mCAQ4GC4752Ucds9YyV7Vb8oOB4KiekG+BZiyaPSse/+QTkx7VA+q2wy5Kc7bTXGsK1BJKZ8nZf8sBcVVuypxX8ZJjK3x4h6ChD396S6SWGu9KOR5Gws+4xT6nE1xBZfzVt/pSA=; bm_sz=993B9150C6357D1453561ED8533D948C~QAAQ1DLFF1EGQC9oAQAAWW78Pvljdn9YU6uwR/wj13nNtKbJAd1tz/XtkSB4D0GzZ0sB6xr9f6lmedf6cPJTA7GFYjVV7H4/3px/zWyNeY73uc1wGoIs4b2EDW9ft+53/wIzZavesl7pmzhBMiCA5GKqJ4WFF9oO4+EP3gtOxVKIKMDjVP8J/UAn2a3coAOZ0zs=; THD_SESSION=; AMCV_F6421253512D2C100A490D45%40AdobeOrg=-894706358%7CMCIDTS%7C17908%7CMCMID%7C81084504873679911179154344636083105898%7CMCAID%7CNONE%7CMCOPTOUT-1547252157s%7CNONE%7CvVersion%7C2.3.0; THD_PERSIST=C4%3D641%2BRohnert%20Pk%20-%20Rohnert%20Park%2C%20CA%2B%3A%3BC4_EXP%3D1578780959%3A%3BC24%3D94928%3A%3BC24_EXP%3D1578780959%3A%3BC34%3D32.1%3A%3BC34_EXP%3D1547331358%3A%3BC39%3D1%3B7%3A00-20%3A00%3B2%3B6%3A00-22%3A00%3B3%3B6%3A00-22%3A00%3B4%3B6%3A00-22%3A00%3B5%3B6%3A00-22%3A00%3B6%3B6%3A00-22%3A00%3B7%3B6%3A00-22%3A00%3A%3BC39_EXP%3D1547248559; ResonanceSegment=1; signInBubble=true; s_dfa=homedepotprod%2Chomedepotglobaldev; ats-cid-AM-141099-sid=46341048; LPSID-31564604=tgZo3ZtbTmaR_Y-e3nvjcQ; bm_mi=21A83C44D89B199539D14330195C2F99~mGutnDw9T1dVaFjCLHwYLZw3kAOjDjotPSLR4WpSqqHiFQpXWyR4sK+9B3XaCHm6PTzxe40CC15/SSfwKqFR69zJ9MNcL+Ikczcr4Mq4STJ8E4DGF8ozcpOICIx7wUHr4bN9S6EoHVGMsDpO6tntSLxWq4i2U3SDGUXpFjyWEHVYJoMil8dK0D1+BduiBRPiryGLnSu00wtXOnXDXIgSEETU/dWSRtwRyDwYSIA02+JU+MGUWuObfgriro9Zd2tl; bm_sv=176F5E5B2FBE5E8295D8CCE4214111A6~lvtdTZIv6RPPE+bbvi5sIhT//R4DBFahLwcKcw/HA61WomPuVKOI4uq9Fr0BJTsx7o1U57Vsw3iSq+JvBAadGMOiQjWILK6R/pWM/LHXLaRQCnWoMbQbe1Is1Zc4ZKe4wStAcGNCCi2+WT4udxpsK3vuYFnaBUlGjr1uTmEGZ/o=; forterToken=2a23633194f246a7871e5c4f3f1546d8_1547247463626_21_UDF43_6; s_pers=%20s_nr%3D1547247468412-Repeat%7C1578783468412%3B%20s_dslv%3D1547247468416%7C1641855468416%3B%20s_dslv_s%3DLess%2520than%25201%2520day%7C1547249268416%3B%20productnum%3D98%7C1549839468424%3B; IR_8154=1547247468480%7C0%7C1547247402943%7C308SnG1k%3Ax6NRl41WlS2czhgUkgycW2FszrdRM0; IR_PI=1538887702311.ypfh9yayldq%7C1547333868480; s_sess=%20stsh%3D%3B%20s_pv_pName%3Dgift%2520card%253Ebalance%2520check%3B%20s_pv_pType%3Dgift%2520card%3B%20s_pv_cmpgn%3D%3B%20s_cc%3Dtrue%3B%20s_sq%3Dhomedepotprod%25252Chomedepotglobaldev%253D%252526c.%252526a.%252526activitymap.%252526page%25253Dgift%25252520card%2525253Ebalance%25252520check%252526link%25253DCheck%25252520Balance%252526region%25253Dapp%252526pageIDType%25253D1%252526.activitymap%252526.a%252526.c%252526pid%25253Dgift%25252520card%2525253Ebalance%25252520check%252526pidt%25253D1%252526oid%25253Dfunctionpr%25252528%25252529%2525257B%2525257D%252526oidt%25253D2%252526ot%25253DSUBMIT%3B",
"cache-control": "no-cache",
}
)
payload = {
"GiftCardsRequest": {
"cardNumber": kwargs["card_number"],
"pinNumber": kwargs["pin"],
"reCaptcha": captcha_resp["solution"]["gRecaptchaResponse"],
}
}
logger.info(f"Fetching balance from API")
try:
resp = session.post(self.api_endpoint, json=payload, timeout=5)
except requests.exceptions.RequestException as e:
raise RuntimeError(f"Error on API post: {e}")
if resp.status_code != 200:
raise RuntimeError(
f"Failed to get valid response from API (status code {resp.status_code})"
)
print(resp.text)
quit()
def check_balance(self, **kwargs):
if self.validate(kwargs):
logger.info("Checking balance for card: {}, exp {}/{}".format(
kwargs["card_number"], kwargs["exp_month"],
kwargs["exp_year"]))
return self.scrape({
"ctl00$ctl00$FullContent$MainContent$tbNumber":
kwargs["card_number"],
"ctl00$ctl00$FullContent$MainContent$tbExpDate":
kwargs["exp_month"] + kwargs["exp_year"],
"ctl00$ctl00$FullContent$MainContent$tbCid":
kwargs["cvv"],
})
def scrape(self, cards_chunk):
session = requests.Session()
session.headers.update({
"user-agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
"accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"accept-encoding": "gzip, deflate, br",
"accept-language": "en-US,en;q=0.9",
})
# Get balance page in session to retrieve cookies
session.get(self.website_url)
# Assemble a string for JSON request from cards chunk taken
cards_str = ""
for i, card in enumerate(cards_chunk):
if i > 0:
cards_str += ","
cards_str += card["card_number"] + ":" + card["pin"]
payload = {
"gift_card_numbers": cards_str,
"action": "checkCertificateBalance",
"country": "US",
"lang_locale": "en_US",
}
session.headers.update(
{"content-type": "application/x-www-form-urlencoded"})
logger.info(f"Fetching balance from API")
try:
resp = session.post(self.api_endpoint, data=payload, timeout=5)
except requests.exceptions.RequestException as e:
raise RuntimeError(f"Error on API post: {e}")
if resp.status_code != 200:
raise RuntimeError(
f"Failed to get valid response from API (status code {resp.status_code})"
)
print(resp.text)
quit()
def scrape(self, **kwargs):
logger.info("Solving reCAPTCHA (~30s)")
captcha_solver = CaptchaSolver(api_key=config.ANTI_CAPTCHA_KEY)
# Site key obtained from: https://www.homedepot.com/mycheckout/assets/react/giftcard.bundle.1.2302.0.js
captcha_resp = captcha_solver.solve_recaptcha(
self.website_url, "6LfEHBkTAAAAAHX6YgeUw9x1Sutr7EzhMdpbIfWJ")
if captcha_resp["errorId"] != 0:
raise RuntimeError(
f"Unable to solve reCAPTCHA ({captcha_resp['errorDescription']})"
)
payload = {
"GiftCardsRequest": {
"cardNumber": kwargs["card_number"],
"pinNumber": kwargs["pin"],
"reCaptcha": captcha_resp["solution"]["gRecaptchaResponse"],
}
}
logger.info("Fetching balance from API")
try:
resp = requests.post(self.api_endpoint,
json=payload,
headers=HEADERS)
if resp.status_code != 200:
raise RuntimeError(
f"Invalid API response (status code {resp.status_code})")
result = deep_get(resp.json(), "giftCards.giftCard")
if result is None:
raise RuntimeError(
f"Invalid API response: unable to find giftCard key in JSON response"
)
err_code = deep_get(result, "errorCode")
if err_code:
err_desc = deep_get(result, "description")
raise RuntimeError(
f"Failed to retrieve balance from API: {err_desc} ({err_code})"
)
initial_balance = deep_get(result, "originalAmount")
avail_balance = deep_get(result, "availableAmount")
logger.info(f"Success! Card balance: {avail_balance}")
return {
"initial_balance": initial_balance,
"available_balance": avail_balance,
}
except requests.exceptions.RequestException as e:
raise RuntimeError(f"Error on API post: {e}")
except JSONDecodeError:
raise RuntimeError("Failed to parse API response as JSON")
def scrape(self, fields):
# Open Selenium browser
browser = webdriver.Chrome()
logger.info("Fetching balance check page")
browser.get(self.website_url)
logger.info("Filling balance check form")
for field, val in fields.items():
try:
browser.find_element_by_id(field).send_keys(val)
time.sleep(1)
except NoSuchElementException:
browser.close()
raise RuntimeError(f"Unable to find '{field}' field on page")
# Click submit button
browser.find_element_by_id("brandLoginForm_button").click()
# Wait for page to load
try:
WebDriverWait(browser, 3).until(
EC.presence_of_element_located((By.ID, "Avlbal")))
except TimeoutException:
browser.close()
raise RuntimeError("Balance page took too long to load")
logger.info("Obtaining card information")
try:
avail_balance = browser.find_element_by_id("Avlbal").text
except NoSuchElementException:
browser.close()
raise RuntimeError("Could not find available card balance")
try:
initial_balance = (browser.find_element_by_class_name(
"rightSide").find_element_by_tag_name("span").text)
except NoSuchElementException:
browser.close()
raise RuntimeError("Could not find initial card balance")
browser.close()
logger.info(f"Success! Card balance: {avail_balance}")
return {
"initial_balance": initial_balance,
"available_balance": avail_balance
}
async def scrape(self, fields):
browser = await pyppeteer.launch(handleSIGINT=False,
handleSIGTERM=False,
handleSIGHUP=False)
page = await browser.newPage()
logger.info("Fetching balance check page")
await page.goto(self.website_url)
logger.info("Filling balance check form")
await page.type("#Card_Number", fields["card_number"])
await page.type("#Card_Pin", fields["pin"])
logger.info("Requesting balance")
await page.click("#CheckBalance button")
await page.waitForSelector(".fetch_balance_value", {"timeout": 10000})
avail_balance = await page.querySelectorEval(
".fetch_balance_value", "(node => node.innerText)")
logger.info("Success! Card balance: {}".format(avail_balance))
return {"available_balance": avail_balance}
def scrape(self, **kwargs):
session = requests.Session()
session.headers.update({
"User-Agent": self.ua.random, # fake UA
"accept": "application/json",
"origin": "https://www.bestbuy.com",
"content-type": "application/json",
"referer": "https://www.bestbuy.com/digitallibrary/giftcard",
"accept-encoding": "gzip, deflate, br",
"accept-language": "en-US,en;q=0.9",
"cache-control": "no-cache",
})
payload = f'{{"cardNumber":"{kwargs["card_number"]}","pin":"{kwargs["pin"]}"}}'
logger.info(f"Fetching balance from API")
try:
resp = session.post(self.website_url, data=payload, timeout=5)
except requests.exceptions.RequestException as e:
raise RuntimeError(f"Error on API post: {e}")
if resp.status_code != 200:
raise RuntimeError(
f"Failed to get valid response from API (status code {resp.status_code})"
)
# TODO: Sometimes does not output balance or throw error, not sure why. Happens with IP temp block
try:
avail_balance = resp.json()["balance"]
except:
raise RuntimeError("Could not parse balance from JSON response")
logger.info(f"Success! Card balance: {avail_balance}")
# TODO: figure out cleaner way to do this, feature in main?
self.num_runs += 1
# Not sure exactly what sweet spot is but IP blocks at 10 min, 15 min seems good
seconds = 3 if self.num_runs % 9 else 60 * 15 + 5
logger.info(
f"Ran {self.num_runs} times. Sleeping {seconds} seconds before trying next..."
)
time.sleep(seconds)
return {"balance": avail_balance}
def scrape(self, fields):
# Open Selenium browser
browser = webdriver.Chrome()
browser.set_window_size(600, 800)
logger.info("Fetching balance check page")
browser.get(self.website_url)
try:
form = browser.find_elements_by_tag_name("form")[1]
except (NoSuchElementException, IndexError):
raise RuntimeError("Unable to find login form on page")
logger.info("Filling login form 1/2")
try:
form.find_element_by_name("accountNumber").send_keys(
fields["accountNumber"])
time.sleep(1)
except NoSuchElementException:
browser.close()
raise RuntimeError(f"Unable to find 'accountNumber' field on page")
# Click continue button
form.find_element_by_css_selector("input[type='submit']").click()
# Wait for page to load
try:
WebDriverWait(browser, 3).until(
EC.presence_of_element_located((By.ID, "login-form")))
except TimeoutException:
browser.close()
raise RuntimeError("Login page took too long to load")
try:
form = browser.find_element_by_id("login-form")
except NoSuchElementException:
browser.close()
raise RuntimeError("Unable to find login form on page")
logger.info("Solving CAPTCHA (~10s)")
# Extract CAPTCHA image from page
captcha_b64 = get_image_b64_by_id(browser, "captchaImg")
captcha_solver = CaptchaSolver(api_key=config.ANTI_CAPTCHA_KEY)
captcha = captcha_solver.solve_image_b64(captcha_b64)
if captcha["errorId"] != 0:
browser.close()
raise RuntimeError("Unable to solve CAPTCHA ({})".format(
captcha["errorDescription"]))
logger.info("Filling login form 2/2")
try:
form.find_element_by_name("cv2").send_keys(fields["cv2"])
time.sleep(1)
except NoSuchElementException:
browser.close()
raise RuntimeError("Unable to find 'cv2' field on page")
try:
form.find_element_by_id(
"_MultiStageFSVpasswordloginresponsive_WAR_cardportalresponsive_captchaText"
).send_keys(captcha["solution"]["text"])
time.sleep(1)
except NoSuchElementException:
browser.close()
raise RuntimeError("Unable to find CAPTCHA field on page")
# Click continue button
form.find_element_by_css_selector("input[type='submit']").click()
# Wait for page to load
try:
WebDriverWait(browser, 3).until(
EC.presence_of_element_located((By.ID, "cardBalanceInfo")))
except TimeoutException:
browser.close()
raise RuntimeError("Balance page took too long to load")
logger.info("Obtaining card information")
try:
avail_balance = browser.find_element_by_class_name(
"cardBalanceText").text
except NoSuchElementException:
browser.close()
raise RuntimeError("Could not find available card balance")
browser.close()
logger.info(f"Success! Card balance: {avail_balance}")
return {"initial_balance": None, "available_balance": avail_balance}
def scrape(self, fields):
session = requests.Session()
session.headers.update({"User-Agent": config.USER_AGENT})
logger.info("Fetching balance check page")
resp = session.get(self.website_url)
if resp.status_code != 200:
raise RuntimeError(
f"Failed to GET website (status code {resp.status_code}")
page_html = BeautifulSoup(resp.content, features="html.parser")
form = page_html.find("form")
if not form:
raise RuntimeError("Unable to find balance check form")
endpoint = f"{self.website_url}{form['action']}"
# These fields are present on GS balance check page, does not work without including them
fields['__EVENTTARGET'] = ''
fields['__EVENTARGUMENT'] = ''
fields['__LASTFOCUS'] = ''
fields['__VIEWSTATE'] = page_html.find("input",
id='__VIEWSTATE')['value']
fields['__VIEWSTATEGENERATOR'] = page_html.find(
"input", id='__VIEWSTATEGENERATOR')['value']
recaptcha_field = page_html.find("div", class_="g-recaptcha")
if not recaptcha_field:
raise RuntimeError("Unable to find reCAPTCHA")
site_key = recaptcha_field["data-sitekey"]
logger.info("Solving reCAPTCHA (~30s)")
captcha_resp = captcha_solver.solve_recaptcha(self.website_url,
site_key)
if captcha_resp["errorId"] != 0:
raise RuntimeError(
f"Unable to solve reCAPTCHA ({captcha_resp['errorDescription']})"
)
fields["g-recaptcha-response"] = captcha_resp["solution"][
"gRecaptchaResponse"]
logger.info("Fetching card balance")
session.headers.update({
# Not necessary for this merchant
})
form_resp = session.post(endpoint, data=fields)
if form_resp.status_code != 200:
raise RuntimeError(
f"Failed to retrieve card balance (status code {form_resp.status_code})"
)
balance_html = BeautifulSoup(form_resp.content, features="html.parser")
try:
avail_balance = balance_html.find("span",
class_='balancePrice').text
except:
# GS prunes old depleted cards from its system and throws an 'invalid' error
# Set these as -1 to notate they gave invalid error but are likely actually zero balance
if balance_html.find(
text='The Gift Card number entered is invalid.'):
avail_balance = '-1'
elif balance_html.find(
'span',
id=
'BaseContentPlaceHolder_mainContentPlaceHolder_recaptchaMessage'
):
# Message on screen: "The code you entered is invalid."
# CAPTCHA answer invalid, retry
raise RuntimeError(
'Invalid CAPTCHA answer supplied! Trying again...')
else:
raise RuntimeError(
'Could not find balance on retrieved page for unknown reason'
)
logger.info(f"Success! Card balance: {avail_balance}")
return ({"balance": avail_balance})
def check_balance(self, **kwargs):
if self.validate(kwargs):
logger.info(f"Checking balance for card: {kwargs['card_number']}")
return self.scrape(number=kwargs["card_number"], pin=kwargs["pin"])
def main():
providers_help = "\n".join(
[" - {}".format(p_name) for p_name in providers.keys()])
parser = ArgumentParser(
formatter_class=RawTextHelpFormatter,
description=f"""Check gift card balances for a variety of providers.
Supported providers:
{providers_help}
Requires an Anti-CAPTCHA API key for providers with CAPTCHAs.
Get one here: https://anti-captcha.com
Configure your key by setting the ANTI_CAPTCHA_KEY environment variable.
Your INPUT_CSV should be formatted as follows:
- A header row is required
- Each column should contain a parameter required by
the specified provider
Example (for the 'blackhawk' provider):
-------------------------------------------------
| card_number | exp_month | exp_year | cvv |
|------------------|-----------|----------|-----|
| 4111111111111111 | 12 | 24 | 999 |
-------------------------------------------------
If you find this tool useful, consider buying a coffee for the author:
https://stevenmirabito.com/kudos""",
)
parser.add_argument("-v",
"--version",
action="version",
version=f"%(prog)s {version.__version__}")
parser.add_argument(
"provider",
metavar="PROVIDER",
type=str.lower,
help="Name of balance check provider",
)
parser.add_argument("input",
metavar="INPUT_CSV",
type=str,
help="Path to input CSV")
parser.add_argument(
"--output",
"-o",
metavar="OUTPUT_CSV",
type=str,
help=("Path to output CSV (optional; default:"
"add/overwrite\nbalance columns on input CSV)"),
)
args = parser.parse_args()
in_filename = path.abspath(args.input)
out_filename = in_filename if not args.output else path.abspath(
args.output)
if args.provider not in providers:
logger.fatal(f"Unknown provider: '{args.provider}'")
sys.exit(1)
provider = providers[args.provider]
max_workers = (provider.max_workers
if hasattr(provider, "max_workers") else config.MAX_WORKERS)
provider_allows_chunks = True if hasattr(provider,
"max_simultaneous") else False
futures = {}
results = []
retries = {}
with ThreadPoolExecutor(max_workers=max_workers) as executor:
try:
with open(in_filename, newline="") as input_csv:
reader = csv.DictReader(input_csv)
_chunk = []
for i, card_data in enumerate(reader):
# Add the card details to the result
results.append(card_data)
# Some balance checkers accept multiple cards so prepare to send chunks
# !! NEEDS WORK, UNFINISHED
if provider_allows_chunks:
_chunk.append(card_data)
if (
i + 1
) % provider.max_simultaneous: # If end of chunk, send to schedule...
# Schedule balance check
future = executor.submit(provider.check_balance,
_chunk)
futures[future] = i
_chunk = [] # Clear chunk at end
else:
future = executor.submit(provider.check_balance,
**card_data)
futures[future] = i
# Done reading input file and scheduling
logger.info(f"Read {len(results)} cards from file '{in_filename}'")
except (OSError, IOError) as err:
logger.fatal(f"Unable to open input file '{in_filename}': {err}")
sys.exit(1)
except Exception as e:
logger.fatal(f"Unexpected error: {e}")
sys.exit(1)
# While there are still tasks queued, jump back in (handles retries)
while futures:
# Update progress bar as tasks complete
for future in tqdm(as_completed(futures),
total=len(futures),
leave=False):
idx = futures.pop(future)
try:
balance_info = future.result()
except Exception as e:
# Log the first column value as an ID (usually card number)
card_id = next(iter(results[idx].values()))
# Attempt to schedule retry
if idx in retries:
retries[idx] += 1
if retries[idx] > config.RETRY_TIMES:
# Out of retries, permanent failure
logger.error(
f"Failed to balance check {card_id} (out of retries). Last error: {e}"
)
else:
executor.submit(logger.error,
"error occurred",
exc_info=sys.exc_info())
retries[idx] = 1
logger.warning(
"RETRY {}/{}: Failed to balance check {}, will retry later. Error: {}"
.format(retries[idx], config.RETRY_TIMES, card_id,
e))
future = executor.submit(provider.check_balance,
**results[idx])
futures[future] = idx
else: # Successful balance(s) returned
# !! NEEDS WORK, UNFINISHED
if provider_allows_chunks:
# List of balances from chunk of cards returned
for i, balance_info in enumerate(balances_info):
results[idx] = dict(results[idx], **balance_info)
# If not on last cards balance info...
if len(balance_info) - 1 != i:
idx += 1
else:
# Single balance returned
results[idx] = dict(results[idx], **balance_info)
try:
with open(out_filename, "w", newline="") as output_csv:
logger.info(f"Writing CSV output to {out_filename}...")
fieldnames = results[0].keys()
writer = csv.DictWriter(output_csv, fieldnames=fieldnames)
writer.writeheader()
for row in results:
writer.writerow(row)
logger.info(f"Output written to: {out_filename}")
except (OSError, IOError) as err:
logger.fatal(f"Unable to open output file '{in_filename}': {err}")
sys.exit(1)
except Exception as e:
logger.fatal(f"Unexpected error: {e}")
sys.exit(1)
def scrape(self, fields):
session = requests.Session()
session.headers.update({"User-Agent": config.USER_AGENT})
logger.info("Fetching balance check page")
resp = session.get(self.website_url)
if resp.status_code != 200:
logger.critical(
f"Failed to GET Simon website (status code {resp.status_code})"
)
sys.exit(1)
page_html = BeautifulSoup(resp.content, features="html.parser")
recaptcha_field = page_html.find("div", class_="g-recaptcha")
if not recaptcha_field:
logger.critical("Unable to find reCAPTCHA")
sys.exit(1)
site_key = recaptcha_field["data-sitekey"]
logger.info("Solving reCAPTCHA (~30s)")
captcha = captcha_solver.solve_recaptcha(self.website_url, site_key)
if captcha["errorId"] != 0:
logger.critical(
f"Unable to solve reCAPTCHA ({captcha['errorDescription']})")
sys.exit(1)
# These fields are present on balance check page, request blocked if not included
fields["__EVENTTARGET"] = ""
fields["__EVENTARGUMENT"] = ""
fields["__LASTFOCUS"] = ""
fields["__VIEWSTATE"] = page_html.find("input",
id="__VIEWSTATE")["value"]
fields["__VIEWSTATEGENERATOR"] = page_html.find(
"input", id="__VIEWSTATEGENERATOR")["value"]
fields["__VIEWSTATEENCRYPTED"] = ""
fields["__EVENTVALIDATION"] = page_html.find(
"input", id="__EVENTVALIDATION")["value"]
fields["ctl00$ctl00$header1$EmailLogin"] = ""
fields["ctl00$ctl00$header1$PasswordLogin"] = ""
fields[
"ctl00$ctl00$FullContent$MainContent$checkBalanceSubmit"] = "CHECK YOUR BALANCE"
fields["g-recaptcha-response"] = captcha["solution"][
"gRecaptchaResponse"]
session.headers.update({
"User-Agent": config.USER_AGENT,
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"Content-Type": "application/x-www-form-urlencoded",
"Referer": self.website_url,
"origin": "https://www.simon.com",
})
logger.info("Fetching card balance")
form_resp = session.post(self.website_url, data=fields)
if form_resp.status_code != 200:
logger.critical(
f"Failed to retrieve card balance (status code {form_resp.status_code})"
)
sys.exit(1)
balance_html = BeautifulSoup(form_resp.content, features="html.parser")
if balance_html.find("label", text="CAPTCHA: Please validate"):
raise RuntimeError("Invalid CAPTCHA answer supplied.")
try:
avail_balance = balance_html.find(
id="ctl00_ctl00_FullContent_MainContent_lblBalance").text
initial_balance = balance_html.find_all(
"td", _class="tblinfo") # [-1].text.strip()
print(initial_balance)
except:
print("DUMP:", resp.text)
raise RuntimeError("Could not find balance on page")
logger.info(f"Success! Card balance: {avail_balance}")
return {
"initial_balance": initial_balance,
"available_balance": avail_balance
}
def main():
providers_help = "\n".join(
[" - {}".format(p_name) for p_name in providers.keys()])
parser = ArgumentParser(
formatter_class=RawTextHelpFormatter,
description="""Check gift card balances for a variety of providers.
Supported providers:
{}
Requires an Anti-CAPTCHA API key for providers with CAPTCHAs.
Get one here: https://anti-captcha.com
Configure your key by setting the ANTI_CAPTCHA_KEY environment variable.
Your INPUT_CSV should be formatted as follows:
- A header row is required
- Each column should contain a parameter required by
the specified provider
Example (for the 'blackhawk' provider):
-------------------------------------------------
| card_number | exp_month | exp_year | cvv |
|------------------|-----------|----------|-----|
| 4111111111111111 | 12 | 24 | 999 |
-------------------------------------------------
If you find this tool useful, consider buying a coffee for the author:
https://stevenmirabito.com/kudos""".format(providers_help),
)
parser.add_argument(
"-v",
"--version",
action="version",
version="%(prog)s {}".format(version.__version__),
)
parser.add_argument(
"provider",
metavar="PROVIDER",
type=str.lower,
help="Name of balance check provider",
)
parser.add_argument("input",
metavar="INPUT_CSV",
type=str,
help="Path to input CSV")
parser.add_argument(
"--output",
"-o",
metavar="OUTPUT_CSV",
type=str,
help=("Path to output CSV (optional; default:"
"add/overwrite\nbalance columns on input CSV)"),
)
args = parser.parse_args()
in_filename = path.abspath(args.input)
out_filename = in_filename
if args.output:
# Separate output path specified
out_filename = path.abspath(args.output)
if args.provider not in providers:
logger.fatal("Unknown provider: '{}'".format(args.provider))
sys.exit(1)
provider = providers[args.provider]
futures = {}
results = []
retries = {}
with ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
try:
with open(in_filename, newline="") as input_csv:
reader = csv.DictReader(input_csv)
for row in reader:
# Add the card details to the result
results.append(row)
idx = len(results) - 1
# Schedule balance check
future = executor.submit(provider.check_balance, **row)
futures[future] = idx
except (OSError, IOError) as err:
logger.fatal("Unable to open input file '{}': {}".format(
in_filename, err))
sys.exit(1)
except Exception as e:
logger.fatal("Unexpected error: {}".format(e))
sys.exit(1)
# While there are still tasks queued, jump back in (handles retries)
while futures:
# Update progress bar as tasks complete
for future in tqdm(as_completed(futures),
total=len(futures),
leave=False):
idx = futures.pop(future)
try:
balance_info = future.result()
except Exception as e:
# Log the first column value as an ID (usually card number)
card_id = next(iter(results[idx].values()))
# Attempt to schedule retry
if idx in retries:
retries[idx] += 1
if retries[idx] > config.RETRY_TIMES:
# Out of retries, permanent failure
logger.error(
"Failed to balance check {} (out of retries). Last error: {}"
.format(card_id, e))
else:
retries[idx] = 1
logger.warning(
"RETRY {}/{}: Failed to balance check {}, retrying. Error: {}"
.format(retries[idx], config.RETRY_TIMES, card_id, e))
future = executor.submit(provider.check_balance,
**results[idx])
futures[future] = idx
else:
# Combine original card details with balance information
results[idx] = dict(results[idx], **balance_info)
try:
with open(out_filename, "w", newline="") as output_csv:
logger.info("Writing output CSV...")
fieldnames = results[0].keys()
writer = csv.DictWriter(output_csv, fieldnames=fieldnames)
writer.writeheader()
for row in results:
writer.writerow(row)
logger.info("Output written to: {}".format(out_filename))
except (OSError, IOError) as err:
logger.fatal("Unable to open output file '{}': {}".format(
in_filename, err))
sys.exit(1)
except Exception as e:
logger.fatal("Unexpected error: {}".format(e))
sys.exit(1)
def scrape(self, **kwargs):
cookies = CookieJar()
opener = request.build_opener(request.HTTPCookieProcessor(cookies))
logger.info("Fetching balance check page")
# Use urllib directly as requests gets blocked
req = request.Request(self.website_url, headers=HEADERS)
resp = opener.open(req)
if resp.status != 200:
raise RuntimeError(
f"Failed to get GameStop website (status code {resp.status})")
page_html = BeautifulSoup(resp.read(), features="html.parser")
recaptcha_el = page_html.find("div", {"data-sitekey": True})
if not recaptcha_el:
raise RuntimeError("Unable to find reCAPTCHA on page")
csrf_el = page_html.find("input", {"name": "csrf_token"})
if not csrf_el:
raise RuntimeError("Unable to find CSRF on page")
site_key = recaptcha_el["data-sitekey"]
csrf_token = csrf_el["value"]
logger.info("Solving reCAPTCHA (~30s)")
captcha_solver = CaptchaSolver(api_key=config.ANTI_CAPTCHA_KEY)
captcha_resp = captcha_solver.solve_recaptcha(self.website_url,
site_key)
if captcha_resp["errorId"] != 0:
raise RuntimeError(
f"Unable to solve reCAPTCHA ({captcha_resp['errorDescription']})"
)
payload = {
"dwfrm_giftCard_balance_accountNumber": kwargs["card_number"],
"dwfrm_giftCard_balance_pinNumber": kwargs["pin"],
"g-recaptcha-response":
captcha_resp["solution"]["gRecaptchaResponse"],
"csrf_token": csrf_token,
}
logger.info("Fetching balance from API")
try:
req = request.Request(self.api_endpoint, headers=HEADERS)
data = urlencode(payload).encode("utf-8")
req.add_header("Content-Type",
"application/x-www-form-urlencoded; charset=UTF-8")
req.add_header("Content-Length", len(data))
resp = opener.open(req, data)
if resp.status != 200:
raise RuntimeError(
f"Invalid API response (status code {resp.status})")
result = json.loads(resp.read().decode(
resp.info().get_param("charset") or "utf-8"))
errors = deep_get(result, "error")
if errors:
err = errors[0]
raise RuntimeError(
f"Failed to retrieve balance from API: {err}")
balance = deep_get(result, "balance")
if balance is None:
raise RuntimeError(
f"Invalid API response: unable to find required key in JSON response"
)
logger.info(f"Success! Card balance: ${balance}")
return {
"balance": f"${balance}",
}
except json.JSONDecodeError:
raise RuntimeError("Failed to parse API response as JSON")
def scrape(self, fields):
session = requests.Session()
session.headers.update({
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Connection": "keep-alive",
})
logger.info("Fetching balance check page")
resp = session.get(self.website_url)
if resp.status_code != 200:
logger.critical(
f"Failed to GET OneVanilla website (status code {resp.status_code})"
)
sys.exit(1)
print(resp.text)
page_html = BeautifulSoup(resp.content, features="html.parser")
action = page_html.find("form")["action"] # brandLoginForm
fields["csrfToken"] = page_html.find("input",
name="csrfToken")["value"]
session.headers.update({
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"Content-Type": "application/x-www-form-urlencoded",
"Referer": self.website_url,
"origin": "https://www.simon.com",
})
logger.info("Fetching card balance")
form_resp = session.post(self.website_url + action[2:], data=fields)
if form_resp.status_code != 200:
logger.critical(
f"Failed to retrieve card balance (status code {form_resp.status_code})"
)
sys.exit(1)
balance_html = BeautifulSoup(form_resp.content, features="html.parser")
try:
avail_balance = balance_html.find("li", id="Avlbal").text.strip()
# initial_balance = balance_html.find("li", text="Original Value:") #[-1].text.strip()
initial_balance = "1"
except:
print("DUMP:", resp.text)
raise RuntimeError("Could not find balance on page")
logger.info(f"Success! Card balance: {avail_balance}")
return {
"initial_balance": initial_balance,
"available_balance": avail_balance
}
def scrape(self, fields):
session = requests.Session()
session.headers.update({"User-Agent": config.USER_AGENT})
fields["X-Requested-With"] = "XMLHttpRequest"
logger.info("Fetching balance check page")
resp = session.get(self.website_url)
if resp.status_code != 200:
logger.critical(
"Failed to GET Blackhawk website (status code {})".format(
resp.status_code))
sys.exit(1)
page_html = BeautifulSoup(resp.content, features="html.parser")
transactions = page_html.find(id="CheckBalanceTransactions")
form = transactions.find("form")
if not form:
logger.critical("Unable to find balance check form")
sys.exit(1)
endpoint = "{}{}".format(self.website_url, form["action"])
token_field = transactions.find(
"input", attrs={"name": "__RequestVerificationToken"})
if not token_field:
logger.critical("Failed to retrieve verification token")
sys.exit(1)
fields["__RequestVerificationToken"] = token_field["value"]
recaptcha_field = transactions.find("div", class_="g-recaptcha")
if not recaptcha_field:
logger.critical("Unable to find reCAPTCHA")
sys.exit(1)
site_key = recaptcha_field["data-sitekey"]
logger.info("Solving reCAPTCHA (~30s)")
captcha = captcha_solver.solve_recaptcha(self.website_url, site_key)
if captcha["errorId"] != 0:
logger.critical("Unable to solve reCAPTCHA ({})".format(
captcha["errorDescription"]))
sys.exit(1)
fields["g-recaptcha-response"] = captcha["solution"][
"gRecaptchaResponse"]
logger.info("Fetching card balance")
session.headers.update({
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.5",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Pragma": "no-cache",
"Referer": "https://mygift.giftcardmall.com/",
"X-Requested-With": "XMLHttpRequest",
})
form_resp = session.post(endpoint, data=fields)
if form_resp.status_code != 200:
logger.critical(
"Failed to retrieve card balance (status code {})".format(
form_resp.status_code))
sys.exit(1)
balance_html = BeautifulSoup(form_resp.content, features="html.parser")
avail_balance = (balance_html.find(
"div", text="Available Balance").parent.find("div",
class_="value").text)
initial_balance = (balance_html.find(
"div", text="Initial Balance").parent.find("div",
class_="value").text)
logger.info("Success! Card balance: {}".format(avail_balance))
return {
"initial_balance": initial_balance,
"available_balance": avail_balance
}
def scrape(self, fields):
session = requests.Session()
session.headers.update({"User-Agent": config.USER_AGENT})
logger.info("Fetching balance check page")
resp = session.get(self.website_url)
if resp.status_code != 200:
raise RuntimeError(
"Failed to GET Spafinder website (status code {})".format(
resp.status_code))
page_html = BeautifulSoup(resp.content, features="html.parser")
inquiry = page_html.find(id="balance-inquiry")
form = inquiry.find("form")
if not form:
raise RuntimeError("Unable to find balance check form")
endpoint = "{}{}".format(self.base_url, form["action"])
# Page has bad HTML, need to search from top level
recaptcha_field = page_html.find("div", class_="g-recaptcha")
if not recaptcha_field:
raise RuntimeError("Unable to find reCAPTCHA")
site_key = recaptcha_field["data-sitekey"]
logger.info("Solving reCAPTCHA (~30s)")
captcha_solver = CaptchaSolver(api_key=config.ANTI_CAPTCHA_KEY)
captcha_resp = captcha_solver.solve_recaptcha(self.website_url,
site_key)
if captcha_resp["errorId"] != 0:
raise RuntimeError("Unable to solve reCAPTCHA ({})".format(
captcha_resp["errorDescription"]))
fields["g-recaptcha-response"] = captcha_resp["solution"][
"gRecaptchaResponse"]
logger.info("Fetching card balance")
form_resp = session.post(endpoint, data=fields)
if form_resp.status_code != 200:
raise RuntimeError(
"Failed to retrieve card balance (status code {})".format(
form_resp.status_code))
resp_html = BeautifulSoup(form_resp.content, features="html.parser")
error_html = resp_html.find("div", class_="alert-danger")
if error_html:
raise RuntimeError("Got error while checking balance: {}".format(
error_html.text))
balance_container = resp_html.find("div", class_="alert-success")
if not balance_container:
raise RuntimeError("Unable to find balance container")
balance_match = re.search("(\d{1,2}\.\d{2})",
balance_container.get_text())
if not balance_match:
raise RuntimeError("Unable to find balance text")
balance = "${}".format(balance_match.group(0))
logger.info("Success! Card balance: {}".format(balance))
return {"balance": balance}
