def buyprd(u):
browser = Browser('firefox')
url = u
browser.visit(url)
# 10|10.5
browser.find_option_by_text(selectOption).first.click()
browser.find_by_name('commit').click()
if browser.is_text_present('item'):
print("Added to Cart")
else:
print("Error")
return
print "checking out"
browser.visit(checkoutUrl)
print "Filling Out Billing Info"
browser.fill("order[billing_name]", namefield)
browser.fill("order[email]", emailfield)
browser.fill("order[tel]", phonefield)
print "Filling Out Address"
browser.fill("order[billing_address]", addressfield)
browser.fill("order[billing_zip]", zipfield)
browser.select("order[billing_state]", statefield)
print "Filling Out Credit Card Info"
browser.select("credit_card[type]", cctypefield)
browser.fill("credit_card[number]", ccnumfield)
browser.select("credit_card[month]", ccmonthfield)
browser.select("credit_card[year]", ccyearfield)
browser.fill("credit_card[verification_value]", cccvcfield)
browser.find_by_css('.terms').click()
print "Submitting Info"
browser.find_by_name('commit').click()
sys.exit(0)
def buyprd(u):
browser = Browser('firefox')
url = u
browser.visit(url)
# 10|10.5
browser.find_option_by_text(selectOption).first.click()
browser.find_by_name('commit').click()
if browser.is_text_present('item'):
print("Added to Cart")
else:
print("Error")
return
print("checking out")
browser.visit(checkoutUrl)
print("Filling Out Billing Info")
browser.fill("order[billing_name]", namefield)
browser.fill("order[email]", emailfield)
browser.fill("order[tel]", phonefield)
print("Filling Out Address")
browser.fill("order[billing_address]", addressfield)
browser.fill("order[billing_zip]", zipfield)
browser.select("order[billing_state]", statefield)
browser.select("order[billing_country], country)
print("Filling Out Credit Card Info")
browser.select("credit_card[type]", cctypefield)
browser.fill("credit_card[number]", ccnumfield)
browser.select("credit_card[month]", ccmonthfield)
browser.select("credit_card[year]", ccyearfield)
browser.fill("credit_card[verification_value]", cccvcfield)
browser.find_by_css('.terms').click()
print("Submitting Info")
browser.find_by_name('commit').click()
sys.exit(0)
class SupremeBot:
def __init__(self, **info):
self.home = 'http://www.supremenewyork.com/'
self.shop_url = 'shop/all/'
self.checkout_url = 'checkout/'
self.info = info
def home_page(self):
self.browser = Browser('chrome', **executable_path)
def find_product(self):
r = requests.get("{}{}{}".format(self.home, self.shop_url,
self.info['category'])).text
soup = bs4.BeautifulSoup(r, "lxml")
all_links = []
needed_links = []
for item in soup.find_all('a', href=True):
all_links.append((item['href'], item.text))
for link in all_links:
if link[1] == self.info['product'] or link[1] == self.info['color']:
needed_links.append(link[0])
self.final_link = list(
set([x for x in needed_links if needed_links.count(x) == 2]))[0]
return self.final_link
def visit_site(self):
self.browser.visit('{}{}'.format(self.home, self.final_link))
self.browser.find_option_by_text(self.info['size']).click()
self.browser.find_by_value('add to cart').click()
def check_out(self):
self.browser.visit('{}{}'.format(self.home, self.checkout_url))
self.browser.find_by_id('order_billing_name').fill(self.info['name'])
self.browser.find_by_id('order_email').fill(self.info['email'])
self.browser.find_by_id('order_tel').fill(self.info['phone'])
self.browser.find_by_id('bo').fill(self.info['addy'])
self.browser.find_by_id('oba3').fill(self.info['apt'])
self.browser.find_by_id('order_billing_zip').fill(
self.info['zipfield'])
self.browser.find_by_id('order_billing_city').fill(self.info['city'])
self.browser.find_by_id('order_billing_name').fill(self.info['name'])
self.browser.find_option_by_text(self.info['country']).click()
self.browser.find_by_id('rnsnckrn').fill(self.info['number'])
card_month = self.browser.find_by_id('credit_card_month').first
card_month.select(self.info['month'])
card_year = self.browser.find_by_id('credit_card_year').first
card_year.select(self.info['year'])
self.browser.find_by_id('orcer').fill(self.info['ccv'])
def buyprd(prdurl):
browser = Browser('firefox')
browser.visit(prdurl)
time.sleep(.5)
print(browser.title)
browser.find_option_by_text(selectOption).first.click()
browser.find_by_name('commit').click()
if browser.is_text_present('item'):
print("Added to Cart")
else:
print("Error")
return
time.sleep(2)
print("checking out")
browser.visit(checkoutUrl)
time.sleep(.5)
print("Filling Out Billing Info")
browser.fill("order[billing_name]", namefield)
browser.fill("order[email]", emailfield)
browser.fill("order[tel]", phonefield)
print("Filling Out Address")
browser.fill("order[billing_address]", addressfield)
browser.fill("order[billing_zip]", zipfield)
browser.select("order[billing_state]", statefield)
print("Filling Out Credit Card Info")
browser.select("credit_card[type]", cctypefield)
browser.fill("credit_card[number]", ccnumfield)
browser.select("credit_card[month]", ccmonthfield)
browser.select("credit_card[year]", ccyearfield)
browser.fill("credit_card[verification_value]", cccvcfield)
browser.find_by_css('.terms').click()
time.sleep(.2)
print("Submitting Info")
browser.find_by_name('commit').click()
time.sleep(1)
browser.driver.save_screenshot('confirmation.png')
print("Exiting...")
time.sleep(2)
sys.exit(0)
class TestViews(unittest.TestCase):
def setUp(self):
""" Test setup """
self.browser = Browser("phantomjs")
self.process = multiprocessing.Process(target=app.run,
kwargs={"port": 8080})
self.process.start()
time.sleep(1)
def test_no_next(self):
self.browser.visit("http://127.0.0.1:8080/")
self.browser.fill("location", "New York")
self.browser.find_option_by_text('Engineering').first.click()
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://127.0.0.1:8080/")
def tearDown(self):
""" Test teardown """
self.process.terminate()
self.browser.quit()
class AppointmenChecker:
def __init__(self, account, startdate=None, enddate=None):
self.account = account
if account == account1[0]:
account_details = account1[1:]
elif account == accont2[0]:
account_details = account2[1:]
else:
raise Exception("Invalid account {}".format(account))
self.username, self.password, self.email, self.phone = account_details
self.startdate = startdate
self.enddate = enddate
self.only_weekends = False
self.b = Browser("chrome")
self.login()
# helper functions
def click_with_retry(self, id):
try:
self.b.find_by_id(id).click()
except:
time.sleep(0.5)
self.b.find_by_id(id).click()
def click_by_text_with_retry(self, text):
try:
self.b.find_by_text(text).click()
except:
time.sleep(0.5)
self.b.find_by_text(text).click()
def fill_with_retry(self, id, content):
try:
self.b.find_by_id(id).fill(content)
except:
time.sleep(0.5)
self.b.find_by_id(id).fill(content)
def select_with_retry(self, id, option):
try:
self.b.find_by_id(id).click()
self.b.find_option_by_text(option).click()
except:
time.sleep(0.5)
self.b.find_by_id(id).click()
self.b.find_option_by_text(option).click()
def next(self):
self.click_with_retry('d-__NextStep')
def login(self):
self.b.visit('https://secure.dol.wa.gov/home')
try:
#b.fill('username', 'mzhang94')
self.b.fill('username', self.username)
except Exception:
self.b.reload()
self.b.fill('username', self.username)
self.b.fill('password', self.password)
self.click_with_retry('btnLogin')
self.click_with_retry('l_k-v-4')
def loadOffice(self, office):
self.next()
self.select_with_retry('d-l1', "Get a license/Id")
self.select_with_retry('d-m1', 'Personal Driver License')
self.next()
self.next()
time.sleep(0.300) # wait for 0.3 second
self.fill_with_retry('d-w2', self.email)
self.fill_with_retry('d-x2', self.email)
self.fill_with_retry('d-03', self.phone)
self.next()
select_option(self.b, 'd-93', 'No')
self.next()
self.select_with_retry('d-14', office)
self.click_with_retry('d-24')
def findAvailableAppointments(self):
appts = []
if self.b.find_by_id('caption2_c-z').first.value != "":
return []
elements = self.b.driver.find_elements_by_class_name("DocFieldLink")
for e in elements:
id = e.get_attribute("id")
time = e.text
if 'AM' not in time and 'PM' not in time:
continue
header_id = "c-{}-CH".format(id.split("-")[1])
date = self.b.find_by_id(header_id).value
appts.append((id, date.replace("\n", " ") + " " + time))
return sorted(appts)
def returnToStartPage(self):
self.b.find_by_text("Cancel")[1].click()
def startScheduleAppointment(self):
self.click_with_retry('d-i')
def makeAppointment(self, appt, office):
appt_id, appt_time = appt
self.click_with_retry(appt_id)
time.sleep(0.3)
self.b.find_by_text('Confirm')[1].click()
time.sleep(0.3)
confirmation_code = self.b.find_by_id('caption2_c-a2').text.split('\n')[0].split()[-1]
message = "Booked appointment at office {} on {} with account {}. Confirmation code: {}"\
.format(office,
datetime.strftime(appt_time, "%A %m/%d/%Y %H:%M:%S"),
self.account,
confirmation_code)
print(message)
send_message(message)
return confirmation_code
def tryMakeAnAppointment(self, office):
self.loadOffice(office)
appts = self.findAvailableAppointments()
appts = filterAppts(appts, self.startdate, self.enddate, self.only_weekends)
if len(appts) > 0:
appt = findBestAppointment(appts)
confirmation_code = self.makeAppointment(appt, office)
return office, appt, confirmation_code
else:
self.returnToStartPage()
return None
def cancelAppointment(self, confirmation_code):
self.click_by_text_with_retry('Cancel an appointment')
self.click_by_text_with_retry('I know my code')
self.fill_with_retry('d-7', confirmation_code)
self.fill_with_retry('d-9', self.email)
self.click_with_retry('d-j') # Find my appointment
self.click_with_retry('d-k') #
self.click_with_retry('c-l') # Cancel appointment
self.click_by_text_with_retry('Yes') # Yes
def close(self):
self.b.quit()
class SupremeBot:
def __init__(self, **info):
self.base_url = "https://supremenewyork.com/"
self.shop_ext = "shop/all/"
self.checkout_ext = "checkout/"
self.info = info
def init_browser(self):
self.b = Browser('chrome')
def find_product(self):
try:
r = requests.get("{}{}{}".format(
self.base_url, self.shop_ext, self.info["category"])).text
print(r)
soup = bs4.BeautifulSoup(r, 'lxml')
temp_tuple = []
temp_link = []
print(soup)
for link in soup.find_all("a", href=True):
temp_tuple.append((link["href"], link.text))
for i in temp_tuple:
if i[1] == self.info["product"] or i[1] == self.info["color"]:
temp_link.append(i[0])
self.final_link = list(
set([x for x in temp_link if temp_link.count(x) == 2]))[0]
print(self.final_link)
except requests.ConnectionError as e:
print("Failed to open url")
def visit_site(self):
self.b.visit("{}{}".format(self.base_url, str(self.final_link)))
self.b.find_option_by_text(self.info["size"]).click()
self.b.find_by_value('add to basket').click()
def checkout_func(self):
self.b.visit("{}{}".format(self.base_url, self.checkout_ext))
self.b.fill("order[billing_name]", self.info["infofield"])
self.b.select("order[billing_country]", self.info["country"])
self.b.fill("order[email]", self.info["emailfield"])
self.b.fill("order[tel]", self.info["phonefield"])
self.b.fill("order[billing_address]", self.info["addressfield"])
self.b.fill("order[billing_city]", self.info["city"])
self.b.fill("order[billing_zip]", self.info["zip"])
self.b.select("credit_card[type]", self.info["card"])
self.b.fill("credit_card[cnb]", self.info["number"])
self.b.select("credit_card[month]", self.info["month"])
self.b.select("credit_card[year]", self.info["year"])
self.b.fill("credit_card[ovv]", self.info["ccv"])
self.b.find_by_css('.terms').click()
self.b.find_by_value("process payment").click()
def main(self):
self.init_browser()
self.find_product()
self.visit_site()
self.checkout_func()
class SupremeBot:
def __init__(self, **info):
self.base = "http://supremenewyork.com/"
self.shop_ext = "shop/all/"
self.checkout_ext = "checkout/"
self.info = info
def init_browser(self):
self.browser = Browser("chrome")
def find_product(self):
try:
req = requests.get("{}{}{}".format(
self.base, self.shop_ext,
self.info["category"])).text # sends back html of page
soup = bs4.BeautifulSoup(
req, "lxml") # allows us to work with html better
temp_link = [
] # list of links that either had product name or color (should only be one element after loop)
for link in soup.find_all("a", href=True):
if link.text == self.info["product"] or link.text == self.info[
"color"]:
temp_link.append(link["href"])
self.final_link = list(
set([x for x in temp_link if temp_link.count(x) == 2]))[0]
return True
except:
False
def visit_site(self):
self.browser.visit("{}{}".format(self.base,
self.final_link)) # go to item page
if self.info["size"] != "":
self.browser.find_option_by_text(
self.info["size"]).click() # select correct size
self.browser.find_by_value("add to cart").click() # add to cart
# wait for item to actually be added to cart
self.browser.is_element_visible_by_xpath("//*[@id='cart']/a[2]",
wait_time=10)
def checkout(self):
self.browser.visit("{}{}".format(
self.base, self.checkout_ext)) # go to checkout page
# fill out order form
self.browser.fill("order[billing_name]", self.info["namefield"])
self.browser.fill("order[email]", self.info["emailfield"])
self.browser.fill("order[tel]", self.info["phonefield"])
self.browser.fill("order[billing_address]", self.info["addressfield"])
self.browser.fill("order[billing_address_2]", self.info["apt"])
self.browser.fill("order[billing_zip]", self.info["zip"])
self.browser.fill("order[billing_city]", self.info["city"])
self.browser.select("order[billing_state]", self.info["state"])
self.browser.select("order[billing_country]", self.info["country"])
self.browser.fill("riearmxa", self.info["number"])
self.browser.select("credit_card[month]", self.info["month"])
self.browser.select("credit_card[year]", self.info["year"])
self.browser.fill("credit_card[meknk]", self.info["ccv"])
# check the terms and conditions and complete order
self.browser.find_by_css(".terms").click()
VehicleColor = 'Your Info' #Must be a common color and in all caps
VehicleLicensePlateNumber = 'Your Info'
LicenseState = 'Your Info'
VehicleOwner = 'Your Info' #This is your name in all caps
Insurance_Co_Name = 'Your Info'
Insurance_PolicyNum = 'Your Info'
Policy_Exp_Date = 'Your Info' #Must be in this format mmddyyyy
browser = Browser('chrome')
url = "https://app4.omnigo.com/Norwich/CESIReportExec/OPR/OPRMain.aspx?IsAuth=1&groupid=102&groupname=PUBLIC+SAFETY"
browser.visit(url)
# Find and click the 'search' button
# Interact with elements
browser.find_by_id('ddPermitType').first
browser.find_option_by_text('STUDENT FULL YEAR PARKING PERMIT').first.click()
browser.find_by_id('ddpermitPeriod').first
browser.find_option_by_text('ANNUAL').first.click()
browser.find_by_id('ddpermitRegistrationType').first
browser.find_option_by_text('RESIDENT STUDENT').first.click()
browser.find_by_id('btnContinue').first.click()
#2nd page
browser.find_by_id('Page_2_Id').first.fill(Studentid)
browser.find_by_id('Page_2_FirstName').first.fill(name)
browser.find_by_id('Page_2_MiddleName').first.fill(middle)
browser.find_by_id('Page_2_LastName').first.fill(last)
browser.find_by_id('ddType').first
browser.find_option_by_text('CONTACT').first.click()
browser.find_by_id('Page_2_HomeAddress').first.fill(address)
browser.find_by_id('Page_2_HomeCity').first.fill(city)
browser.find_by_id('Page_2_HomeState').first.fill(state)
class supremeBot(object):
def __init__(self, **info):
self.base_url = 'http://www.supremenewyork.com/'
self.shop = 'shop/all/'
self.checkout = 'checkout/'
self.info = info
def initializeBrowser(self):
driver = self.info["driver"]
path = helpers.get_driver_path(driver)
if driver == "geckodriver":
self.b = Browser()
elif driver == "chromedriver":
executable_path = {"executable_path": path}
self.b = Browser('chrome', **executable_path)
def findProduct(self):
try:
r = requests.get("{}{}{}".format(self.base_url, self.shop,
self.info['category'])).text
soup = bs.BeautifulSoup(r, 'lxml')
temp_tuple = []
temp_link = []
for link in soup.find_all('a', href=True):
temp_tuple.append((link['href'], link.text))
for i in temp_tuple:
if i[1] == self.info['product'] or i[1] == self.info['color']:
temp_link.append(i[0])
self.final_link = list(
set([x for x in temp_link if temp_link.count(x) == 2]))[0]
return True
except:
return False
def visitSite(self):
self.b.visit("{}{}".format(self.base_url, str(self.final_link)))
self.b.find_option_by_text(self.info['size']).click()
self.b.find_by_value('add to basket').click()
def checkoutFunc(self):
self.b.visit("{}{}".format(self.base_url, self.checkout))
self.b.fill("order[billing_name]", self.info['namefield'])
self.b.fill("order[email]", self.info['emailfield'])
self.b.fill("order[tel]", self.info['phonefield'])
self.b.fill("order[billing_address]", self.info['addressfield'])
self.b.fill("order[billing_city]", self.info['city'])
self.b.fill("order[billing_zip]", self.info['zip'])
self.b.select("order[billing_country]", self.info['country'])
self.b.select("credit_card[type]", self.info['card'])
self.b.fill("credit_card[cnb]", self.info['number'])
self.b.select("credit_card[month]", self.info['month'])
self.b.select("credit_card[year]", self.info['year'])
self.b.fill("credit_card[ovv]", self.info['ccv'])
self.b.find_by_css('.terms').click()
def search_advs(searches):
browser = Browser()
advs_dict = {}
for s in searches:
browser.visit('http://mobile.bg')
try:
browser.find_by_id('ZaplataFooterClose').first.click()
except Exception as e:
pass
browser.find_option_by_text(s['kategoria']).first.click()
browser.find_option_by_text(s['marka']).first.click()
if s['model'] != '':
browser.find_option_by_text(s['model']).first.click()
#go to Podrobno Tursene
browser.find_by_text('Подробно търсене').first.click()
time.sleep(sleep_time)
#close zaplata banner
try:
browser.find_by_id('ZaplataFooterClose').first.click()
print(1)
except Exception as e:
pass
#print ('could not close banner:', e)
browser.find_option_by_text(sortirane).first.click()
if s['dvigatel'] != '':
browser.find_option_by_text(s['dvigatel']).first.click()
if s['godina'][0] != 0:
browser.find_option_by_text('от ' + str(s['godina'][0]) +
' г.').first.click()
if s['godina'][1] != 0:
browser.find_option_by_text('до ' + str(s['godina'][1]) +
' г.').first.click()
if s['region'] != '':
browser.find_option_by_text(s['region']).first.click()
if s['grad'] != '':
browser.find_option_by_text(s['grad']).first.click()
for o in s['opcii']:
browser.find_by_text(o).first.click()
#TODO: add option za chastni obqvi / vsichki obqvi
browser.find_by_value('Т Ъ Р С И').first.click()
advs = get_advs_hrefs(browser)
#for h in advs:
# print(h)
print('total', len(advs))
advs_dict[s['search_name']] = advs
browser.quit()
return advs_dict
class Application:
def __init__(self, master):
self.builder = builder = pygubu.Builder()
builder.add_from_file('bin\\gui.ui')
self.mainwindow = builder.get_object('mainwindow', master)
self.console = builder.get_object('console')
self.productlist = builder.get_object('listbox_products')
builder.connect_callbacks(self)
self.container = Container()
self.builder.import_variables(self.container)
self.load_paymentdata()
def add_productdata(self):
self.productlist.insert(tk.END, [
self.container.productname.get(),
self.container.productcolour.get(),
self.container.productsize.get(),
self.container.productcategory.get()
])
def remove_productdata(self):
index = int(self.productlist.curselection()[0])
self.productlist.delete(index)
def select_productdata(self, event):
index = int(self.productlist.curselection()[0])
value = self.productlist.get(index)
self.container.productname.set(self.productlist.get(index)[0])
self.container.productcolour.set(self.productlist.get(index)[1])
self.container.productsize.set(self.productlist.get(index)[2])
self.container.productcategory.set(self.productlist.get(index)[3])
def load_paymentdata(self):
self.console_print("Loading config.json")
self.config = {}
try:
with open('bin\\config.json', 'r') as f:
self.config = json.load(f)
self.container.value_name.set(self.config['name'])
self.container.value_email.set(self.config['email'])
self.container.value_phone.set(self.config['phone'])
self.container.value_addr1.set(self.config['addr1'])
self.container.value_addr2.set(self.config['addr2'])
self.container.value_addr3.set(self.config['addr3'])
self.container.value_zip.set(self.config['zip'])
self.container.value_city.set(self.config['city'])
self.container.value_country.set(self.config['country'])
self.container.value_cardtype.set(self.config['cardtype'])
self.container.value_cardnumber.set(self.config['cardnumber'])
self.container.value_cardexpmonth.set(self.config['cardexpmonth'])
self.container.value_cardexpyear.set(self.config['cardexpyear'])
self.container.value_cardcode.set(self.config['cardcode'])
self.console_print("Loaded config.json")
except Exception:
self.console_print("Failed to load config.json")
self.update_paymentdata()
def update_paymentdata(self):
self.config.clear()
self.config['name'] = self.container.value_name.get()
self.config['email'] = self.container.value_email.get()
self.config['phone'] = self.container.value_phone.get()
self.config['addr1'] = self.container.value_addr1.get()
self.config['addr2'] = self.container.value_addr2.get()
self.config['addr3'] = self.container.value_addr3.get()
self.config['zip'] = self.container.value_zip.get()
self.config['city'] = self.container.value_city.get()
self.config['country'] = self.container.value_country.get()
self.config['cardtype'] = self.container.value_cardtype.get()
self.config['cardnumber'] = self.container.value_cardnumber.get()
self.config['cardexpmonth'] = self.container.value_cardexpmonth.get()
self.config['cardexpyear'] = self.container.value_cardexpyear.get()
self.config['cardcode'] = self.container.value_cardcode.get()
with open('bin\\config.json', 'w') as f:
json.dump(self.config, f)
self.console_print("Updated config.json")
def console_print(self, msg):
self.console.configure(state="normal")
self.console.insert(tk.END, msg + "\n")
self.console.configure(state="disabled")
self.console.update_idletasks()
def start_bot(self):
self.console_print("Starting bot")
self.browser = Browser("chrome")
#self.browser.set_window_size(0, 0)
self.console_print("Waiting for drop")
#self.productlist.insert(tk.END, ['Work Pant','Black','34','pants'])
self.checkedout = False
while not self.checkedout:
self.product_search()
print("test1")
time.sleep(1)
def product_search(self):
items = 0
if (self.productlist.size() < 1):
return
for product in self.productlist.get(0, self.productlist.size()):
found_flag = False
self.console_print("Searching for " + product[0])
try:
req = requests.get(mainUrl + product[3]).text
soup = BeautifulSoup(req, 'html.parser')
for div in soup.find_all('div', 'turbolink_scroller'):
name_flag = False
for a in div.find_all('a', href=True, text=True):
print(a.text)
if product[0] in a.text:
name_flag = True
if name_flag and product[1] in a.text:
productUrl = baseUrl + a['href']
self.console_print("Found " + product[0])
self.browser.visit(productUrl)
try:
self.browser.find_option_by_text(
product[2]).first.click()
self.browser.find_by_name('commit').click()
self.console_print("Added " + product[0] +
" size " + product[2] +
" to cart")
items = items + 1
time.sleep(0.1)
except Exception:
self.console_print(
'Error whilst adding specified ' +
product[0] + ' size to cart.')
break
except Exception:
self.console_print("Failed to find " + product[0])
if items > 0:
self.checkout()
def checkout(self):
#Fill payment data
self.browser.visit(checkoutUrl)
self.browser.fill('order[billing_name]', self.config['name'])
self.browser.fill('order[email]', self.config['email'])
self.browser.fill('order[tel]', self.config['phone'])
self.browser.fill('order[billing_address]', self.config['addr1'])
self.browser.fill('order[billing_address_2]', self.config['addr2'])
self.browser.fill('order[billing_address_3]', self.config['addr3'])
self.browser.fill('order[billing_zip]', self.config['zip'])
self.browser.fill('order[billing_city]', self.config['city'])
self.browser.select('order[billing_country]', self.config['country'])
self.browser.select('credit_card[type]', self.config['cardtype'])
self.browser.fill('credit_card[cnb]', self.config['cardnumber'])
self.browser.select('credit_card[month]', self.config['cardexpmonth'])
self.browser.select('credit_card[year]', self.config['cardexpyear'])
try:
self.browser.fill('credit_card[vval]', self.config['cardcode'])
except:
self.browser.fill('credit_card[ovv]', self.config['cardcode'])
#Accept terms
self.browser.find_by_css('.iCheck-helper')[1].click()
#Checkout
self.browser.find_by_name('commit').click()
self.checkedout = True
from splinter import Browser
from bs4 import BeautifulSoup
from config import user, password, the_assignment
import time
executable_path = {'executable_path': 'Type the path of your chromedriver here'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit("https://bootcampspot.com")
browser.find_by_css("#emailAddress").type(user)
browser.find_by_css("#password").type(password)
browser.find_by_css("button.btn-submit").click()
time.sleep(2)
browser.find_by_css("td.col-md-3:nth-child(3)").click()
browser.find_by_css("span").click()
browser.find_by_xpath(".//a[contains(@href,'gradebook')]").click()
browser.find_option_by_text(the_assignment).first.click()
submissions = browser.find_by_css("a.text-link")
#use files
student_notes = open("student_notes.txt","a")
homework_file = open("homework_files.txt", "a")
for x in range(0, len(submissions)):
print(x)
student = ""
notes = ""
browser.find_by_css("a.text-link")[x].click()
student = browser.find_by_xpath(".//*[contains(@href,'students/')]")
student_notes.write(student.text + "\n")
student_notes.write("--" * 12 + "\n")
notes = browser.find_by_css("div.col-xs-12>p:nth-child(3)")
def nba_scrape():
# Regular Season Shot Zones and Percentages
reg_basic_url = "http://stats.nba.com/players/traditional/?sort=PTS&dir=-1&Season=2017-18&SeasonType=Regular%20Season"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(reg_basic_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
reg_basic_html = browser.html
reg_basic_soup = BeautifulSoup(reg_basic_html, 'html.parser')
reg_basic_table = reg_basic_soup.findAll("tbody")[0].findAll("tr")
reg_basic_list = []
for players in reg_basic_table:
player = players.findAll('td')
try:
reg_basic_dict = {
"name": player[1].text,
"team": player[2].text,
"age": player[3].text,
"gp": player[4].text,
"wins": player[5].text,
"losses": player[6].text,
"min": player[7].text,
"pts": player[8].text,
"fgm": player[9].text,
"fga": player[10].text,
"fgp": player[11].text,
"3pm": player[12].text,
"3pa": player[13].text,
"3pp": player[14].text,
"ftm": player[15].text,
"fta": player[16].text,
"ft%": player[17].text,
"orb": player[18].text,
"drb": player[19].text,
"reb": player[20].text,
"ast": player[21].text,
"tov": player[22].text,
"stl": player[23].text,
"blk": player[24].text,
"pf": player[25].text,
"fp": player[26].text,
"dd2": player[27].text,
"td3": player[28].text,
"pm": player[29].text
}
except:
continue
reg_basic_list.append(reg_basic_dict)
# print (len(reg_basic_list))
# print(reg_basic_list)
# In[21]:
# Advanced Regular Season Stats from Basketball Reference
reg_adv_url = "http://stats.nba.com/players/advanced/?sort=GP&dir=-1&Season=2017-18&SeasonType=Regular%20Season"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(reg_adv_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
reg_adv_html = browser.html
reg_adv_soup = BeautifulSoup(reg_adv_html, 'html.parser')
reg_adv_table = reg_adv_soup.findAll("tbody")[0].findAll("tr")
reg_adv_list = []
for players in reg_adv_table:
player = players.findAll('td')
try:
reg_adv_dict = {
"name": player[1].text,
"team": player[2].text,
"age": player[3].text,
"gp": player[4].text,
"wins": player[5].text,
"losses": player[6].text,
"min": player[7].text,
"ortg": player[8].text,
"drtg": player[9].text,
"netrtg": player[10].text,
"astp": player[11].text,
"ato": player[12].text,
"orbp": player[13].text,
"drbp": player[14].text,
"toratio": player[15].text,
"efg": player[16].text,
"tsp": player[17].text,
"usg": player[18].text,
"pace": player[19].text,
"pie": player[20].text
}
except:
continue
reg_adv_list.append(reg_adv_dict)
# print (len(reg_adv_list))
# print(reg_adv_list)
# Basic Playoff Stats from Basketball Reference
po_basic_url = "http://stats.nba.com/players/traditional/?sort=PTS&dir=-1&Season=2017-18&SeasonType=Playoffs"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(po_basic_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
po_basic_html = browser.html
po_basic_soup = BeautifulSoup(po_basic_html, 'html.parser')
po_basic_table = po_basic_soup.findAll("tbody")[0].findAll("tr")
po_basic_list = []
for players in po_basic_table:
player = players.findAll('td')
try:
po_basic_dict = {
"name": player[1].text,
"team": player[2].text,
"age": player[3].text,
"gp": player[4].text,
"wins": player[5].text,
"losses": player[6].text,
"min": player[7].text,
"pts": player[8].text,
"fgm": player[9].text,
"fga": player[10].text,
"fgp": player[11].text,
"3pm": player[12].text,
"3pa": player[13].text,
"3pp": player[14].text,
"ftm": player[15].text,
"fta": player[16].text,
"ft%": player[17].text,
"orb": player[18].text,
"drb": player[19].text,
"reb": player[20].text,
"ast": player[21].text,
"tov": player[22].text,
"stl": player[23].text,
"blk": player[24].text,
"pf": player[25].text,
"fp": player[26].text,
"dd2": player[27].text,
"td3": player[28].text,
"pm": player[29].text
}
except:
continue
po_basic_list.append(po_basic_dict)
# print (len(po_basic_list))
# print(po_basic_list)
# Advanced Playoff Stats from Basketball Reference
po_adv_url = "http://stats.nba.com/players/advanced/?sort=GP&dir=-1&Season=2017-18&SeasonType=Playoffs"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(po_adv_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
po_adv_html = browser.html
po_adv_soup = BeautifulSoup(po_adv_html, 'html.parser')
po_adv_table = po_adv_soup.findAll("tbody")[0].findAll("tr")
po_adv_list = []
for players in po_adv_table:
player = players.findAll('td')
try:
po_adv_dict = {
"name": player[1].text,
"team": player[2].text,
"age": player[3].text,
"gp": player[4].text,
"wins": player[5].text,
"losses": player[6].text,
"min": player[7].text,
"ortg": player[8].text,
"drtg": player[9].text,
"netrtg": player[10].text,
"astp": player[11].text,
"ato": player[12].text,
"orbp": player[13].text,
"drbp": player[14].text,
"toratio": player[15].text,
"efg": player[16].text,
"tsp": player[17].text,
"usg": player[18].text,
"pace": player[19].text,
"pie": player[20].text
}
except:
continue
po_adv_list.append(po_adv_dict)
# print (len(po_adv_list))
# print(po_adv_list)
# Regular Season Shot Zones and Percentages
reg_shot_zone_url = "https://stats.nba.com/players/shooting/?Season=2017-18&SeasonType=Regular%20Season&DistanceRange=By%20Zone&PerMode=Totals"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(reg_shot_zone_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
reg_shot_zone_html = browser.html
reg_shot_zone_soup = BeautifulSoup(reg_shot_zone_html, 'html.parser')
reg_shot_zone_table = reg_shot_zone_soup.findAll("tbody")[0].findAll("tr")
reg_shot_zone_list = []
for players in reg_shot_zone_table:
player = players.findAll('td')
try:
reg_shot_zone_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"rafgm": player[3].text,
"rafga": player[4].text,
"rafgp": player[5].text,
"paintfgm": player[6].text,
"paintfga": player[7].text,
"paintfgp": player[8].text,
"mrfgm": player[9].text,
"mrfga": player[10].text,
"mrfgp": player[11].text,
"lcfgm": player[12].text,
"lcfga": player[13].text,
"lcfgp": player[14].text,
"rcfgm": player[15].text,
"rcfga": player[16].text,
"rcfgp": player[17].text,
"atbfgm": player[18].text,
"atbfga": player[19].text,
"atbfgp": player[20].text,
}
except:
continue
reg_shot_zone_list.append(reg_shot_zone_dict)
rszl_index = int(len(reg_shot_zone_list) / 2)
reg_shot_zone_list = reg_shot_zone_list[rszl_index:]
# print (len(reg_shot_zone_list))
# print(reg_shot_zone_list)
# Playoff Shot Zones
po_shot_zone_url = "https://stats.nba.com/players/shooting/?Season=2017-18&SeasonType=Playoffs&DistanceRange=By%20Zone&PerMode=Totals"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(po_shot_zone_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
po_shot_zone_html = browser.html
po_shot_zone_soup = BeautifulSoup(po_shot_zone_html, 'html.parser')
po_shot_zone_table = po_shot_zone_soup.findAll("tbody")[0].findAll("tr")
po_shot_zone_list = []
for players in po_shot_zone_table:
player = players.findAll('td')
try:
po_shot_zone_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"rafgm": player[3].text,
"rafma": player[4].text,
"rafgp": player[5].text,
"paintfgm": player[6].text,
"paintfga": player[7].text,
"paintfgp": player[8].text,
"mrfgm": player[9].text,
"mrfga": player[10].text,
"mrfgp": player[11].text,
"lcfgm": player[12].text,
"lcfga": player[13].text,
"lcfgp": player[14].text,
"rcfgm": player[15].text,
"rcfga": player[16].text,
"rcfgp": player[17].text,
"atbfgm": player[18].text,
"atbfga": player[19].text,
"atbfgp": player[20].text,
}
except:
continue
po_shot_zone_list.append(po_shot_zone_dict)
pszl_index = int(len(po_shot_zone_list) / 2)
po_shot_zone_list = po_shot_zone_list[pszl_index:]
# print(len(po_shot_zone_list))
# print(po_shot_zone_list)
# Regular Season 1st Quarter Shots
q1_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Regular%20Season&PerMode=Totals&Period=1"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(q1_shot_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
q1_shot_html = browser.html
q1_shot_soup = BeautifulSoup(q1_shot_html, 'html.parser')
q1_shot_table = q1_shot_soup.findAll("tbody")[0].findAll("tr")
q1_shot_list = []
for players in q1_shot_table:
player = players.findAll('td')
try:
q1_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
q1_shot_list.append(q1_shot_dict)
# print (len(q1_shot_list))
# print(q1_shot_list)
# Regular Season 2nd Quarter Shots
q2_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Regular%20Season&PerMode=Totals&Period=2"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(q2_shot_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
q2_shot_html = browser.html
q2_shot_soup = BeautifulSoup(q2_shot_html, 'html.parser')
q2_shot_table = q2_shot_soup.findAll("tbody")[0].findAll("tr")
q2_shot_list = []
for players in q2_shot_table:
player = players.findAll('td')
try:
q2_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
q2_shot_list.append(q2_shot_dict)
# print (len(q2_shot_list))
# print(q2_shot_list)
# Regular Season 3rd Quarter Shots
q3_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Regular%20Season&PerMode=Totals&Period=3"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(q3_shot_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
q3_shot_html = browser.html
q3_shot_soup = BeautifulSoup(q3_shot_html, 'html.parser')
q3_shot_table = q3_shot_soup.findAll("tbody")[0].findAll("tr")
q3_shot_list = []
for players in q3_shot_table:
player = players.findAll('td')
try:
q3_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
q3_shot_list.append(q3_shot_dict)
# print (len(q3_shot_list))
# print(q3_shot_list)
# Regular Season 4th Quarter Shots
q4_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Regular%20Season&PerMode=Totals&Period=4"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(q4_shot_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
q4_shot_html = browser.html
q4_shot_soup = BeautifulSoup(q4_shot_html, 'html.parser')
q4_shot_table = q4_shot_soup.findAll("tbody")[0].findAll("tr")
q4_shot_list = []
for players in q4_shot_table:
player = players.findAll('td')
try:
q4_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
q4_shot_list.append(q4_shot_dict)
# print (len(q4_shot_list))
# print(q4_shot_list)
# Regular Season OT Quarter Shots
OT_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Regular%20Season&PerMode=Totals&Period=5"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(OT_shot_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
OT_shot_html = browser.html
OT_shot_soup = BeautifulSoup(OT_shot_html, 'html.parser')
OT_shot_table = OT_shot_soup.findAll("tbody")[0].findAll("tr")
OT_shot_list = []
for players in OT_shot_table:
player = players.findAll('td')
try:
OT_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
OT_shot_list.append(OT_shot_dict)
# print (len(OT_shot_list))
# print(OT_shot_list)
# Playoff 1q shots
PO_1q_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Playoffs&PerMode=Totals&Period=1"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(PO_1q_shot_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
PO_1q_shot_html = browser.html
PO_1q_shot_soup = BeautifulSoup(PO_1q_shot_html, 'html.parser')
PO_1q_shot_table = PO_1q_shot_soup.findAll("tbody")[0].findAll("tr")
PO_1q_shot_list = []
for players in PO_1q_shot_table:
player = players.findAll('td')
try:
PO_1q_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
PO_1q_shot_list.append(PO_1q_shot_dict)
# print (len(PO_1q_shot_list))
# print(PO_1q_shot_list)
# 2q playoff shots
PO_2q_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Playoffs&PerMode=Totals&Period=2"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(PO_2q_shot_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
PO_2q_shot_html = browser.html
PO_2q_shot_soup = BeautifulSoup(PO_2q_shot_html, 'html.parser')
PO_2q_shot_table = PO_2q_shot_soup.findAll("tbody")[0].findAll("tr")
PO_2q_shot_list = []
for players in PO_2q_shot_table:
player = players.findAll('td')
try:
PO_2q_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
PO_2q_shot_list.append(PO_2q_shot_dict)
# print (len(PO_2q_shot_list))
# print(PO_2q_shot_list)
# 3Q Shots
PO_3q_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Playoffs&PerMode=Totals&Period=3"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(PO_3q_shot_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
PO_3q_shot_html = browser.html
PO_3q_shot_soup = BeautifulSoup(PO_3q_shot_html, 'html.parser')
PO_3q_shot_table = PO_3q_shot_soup.findAll("tbody")[0].findAll("tr")
PO_3q_shot_list = []
for players in PO_3q_shot_table:
player = players.findAll('td')
try:
PO_3q_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
PO_3q_shot_list.append(PO_3q_shot_dict)
# print (len(PO_3q_shot_list))
# print(PO_3q_shot_list)
# 4Q PO Shot
PO_4q_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Playoffs&PerMode=Totals&Period=4"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(PO_4q_shot_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
PO_4q_shot_html = browser.html
PO_4q_shot_soup = BeautifulSoup(PO_4q_shot_html, 'html.parser')
PO_4q_shot_table = PO_4q_shot_soup.findAll("tbody")[0].findAll("tr")
PO_4q_shot_list = []
for players in PO_4q_shot_table:
player = players.findAll('td')
try:
PO_4q_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
PO_4q_shot_list.append(PO_4q_shot_dict)
# print (len(PO_4q_shot_list))
# print(PO_4q_shot_list)
# PO OT Shots
PO_OT_shot_url = "https://stats.nba.com/players/shots-general/?Season=2017-18&SeasonType=Playoffs&PerMode=Totals&Period=5"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(PO_OT_shot_url)
# browser.find_option_by_text("All").first.click()
time.sleep(10)
PO_OT_shot_html = browser.html
PO_OT_shot_soup = BeautifulSoup(PO_OT_shot_html, 'html.parser')
PO_OT_shot_table = PO_OT_shot_soup.findAll("tbody")[0].findAll("tr")
PO_OT_shot_list = []
for players in PO_OT_shot_table:
player = players.findAll('td')
try:
PO_OT_shot_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"gp": player[3].text,
"g": player[4].text,
"fgfreq": player[5].text,
"fgm": player[6].text,
"fga": player[7].text,
"fgp": player[8].text,
"efgp": player[9].text,
"2fgfreq": player[10].text,
"2fgm": player[11].text,
"2fga": player[12].text,
"2fgp": player[13].text,
"3pfreq": player[14].text,
"3pm": player[15].text,
"3pa": player[16].text,
"3pp": player[17].text
}
except:
continue
PO_OT_shot_list.append(PO_OT_shot_dict)
# print (len(PO_OT_shot_list))
# print(PO_OT_shot_list)
# Regular Season Clutch Stats (Last 5 minutes in a close game)
reg_clutch_url = "https://stats.nba.com/players/clutch-traditional/?sort=GP&dir=-1&Season=2017-18&SeasonType=Regular%20Season&PerMode=Totals"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(reg_clutch_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
reg_clutch_html = browser.html
reg_clutch_soup = BeautifulSoup(reg_clutch_html, 'html.parser')
reg_clutch_table = reg_clutch_soup.findAll("tbody")[0].findAll("tr")
reg_clutch_list = []
for players in reg_clutch_table:
player = players.findAll('td')
try:
reg_clutch_dict = {
"name": player[1].text,
"team": player[2].text,
"age": player[3].text,
"gp": player[4].text,
"wins": player[5].text,
"losses": player[6].text,
"min": player[7].text,
"pts": player[8].text,
"fgm": player[9].text,
"fga": player[10].text,
"fgp": player[11].text,
"3pm": player[12].text,
"3pa": player[13].text,
"3pp": player[14].text,
"ftm": player[15].text,
"fta": player[16].text,
"ft%": player[17].text,
"orb": player[18].text,
"drb": player[19].text,
"reb": player[20].text,
"ast": player[21].text,
"tov": player[22].text,
"stl": player[23].text,
"blk": player[24].text,
"pf": player[25].text,
"fp": player[26].text,
"dd2": player[27].text,
"td3": player[28].text,
"pm": player[29].text
}
except:
continue
reg_clutch_list.append(reg_clutch_dict)
# print (len(reg_clutch_list))
# print(reg_clutch_list)
# Playoff Clutch Stats (last 5 minutes in a close game)
po_clutch_url = "https://stats.nba.com/players/clutch-traditional/?sort=GP&dir=-1&Season=2017-18&SeasonType=Playoffs&PerMode=Totals"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(po_clutch_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
po_clutch_html = browser.html
po_clutch_soup = BeautifulSoup(po_clutch_html, 'html.parser')
po_clutch_table = po_clutch_soup.findAll("tbody")[0].findAll("tr")
po_clutch_list = []
for players in po_clutch_table:
player = players.findAll('td')
try:
po_clutch_dict = {
"name": player[1].text,
"team": player[2].text,
"age": player[3].text,
"gp": player[4].text,
"wins": player[5].text,
"losses": player[6].text,
"min": player[7].text,
"pts": player[8].text,
"fgm": player[9].text,
"fga": player[10].text,
"fgp": player[11].text,
"3pm": player[12].text,
"3pa": player[13].text,
"3pp": player[14].text,
"ftm": player[15].text,
"fta": player[16].text,
"ft%": player[17].text,
"orb": player[18].text,
"drb": player[19].text,
"reb": player[20].text,
"ast": player[21].text,
"tov": player[22].text,
"stl": player[23].text,
"blk": player[24].text,
"pf": player[25].text,
"fp": player[26].text,
"dd2": player[27].text,
"td3": player[28].text,
"pm": player[29].text
}
except:
continue
po_clutch_list.append(po_clutch_dict)
# print (len(po_clutch_list))
# print(po_clutch_list)
bio_url = "https://stats.nba.com/players/bio/?Season=2017-18&SeasonType=Regular%20Season"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
browser.visit(bio_url)
try:
browser.find_option_by_text("All").first.click()
except:
browser.find_option_by_text("All").first.click()
time.sleep(10)
bio_html = browser.html
bio_soup = BeautifulSoup(bio_html, 'html.parser')
bio_table = bio_soup.findAll("tbody")[0].findAll("tr")
# print(bio_table)
bio_list = []
for players in bio_table:
player = players.findAll('td')
# print (player)
try:
bio_dict = {
"name": player[0].text,
"team": player[1].text,
"age": player[2].text,
"height": player[3].text,
"weight": player[4].text,
"college": player[5].text,
"country": player[6].text,
"draftYear": player[7].text,
"draftRd": player[8].text,
"draftNo": player[9].text
}
except:
continue
bio_list.append(bio_dict)
# total database
nba_stats = {
"bio": bio_list,
"reg_basic": reg_basic_list,
"reg_adv": reg_adv_list,
"po_basic": po_basic_list,
"po_adv": po_adv_list,
"reg_shot_zone": reg_shot_zone_list,
"po_shot_zone": po_shot_zone_list,
"reg_q1_shot": q1_shot_list,
"reg_q2_shot": q2_shot_list,
"reg_q3_shot": q3_shot_list,
"reg_q4_shot": q4_shot_list,
"reg_clutch": reg_clutch_list,
"reg_ot_shot": OT_shot_list,
"po_q1_shot": PO_1q_shot_list,
"po_q2_shot": PO_2q_shot_list,
"po_q3_shot": PO_3q_shot_list,
"po_q4_shot": PO_4q_shot_list,
"po_clutch": po_clutch_list,
"po_ot_shot": PO_OT_shot_list
}
return nba_stats
fillItemTwo = fillItemOne
#Going to website
#for selenium
#driver = webdriver.Chrome("C:\Users\Asimness\Desktop\chrome_driver\chromedriver.exe")
browser = Browser('chrome')
while count < countCheck:
#visit URL
browser.visit('http://www.supremenewyork.com/shop/all/')
#click item
#browser.find_by_xpath('//*[@id="container"]/article[8]/div/a').click()
if (count == 3):
browser.find_by_xpath(fillItemOne).click()
if browser.is_text_present(fillSizeOne):
browser.find_option_by_text(fillSizeOne).first.click()
if (count == 2):
browser.find_by_xpath(fillItemTwo).click()
if browser.is_text_present(fillSizeTwo):
browser.find_option_by_text(fillSizeTwo).first.click()
browser.find_by_xpath('//*[@id="add-remove-buttons"]/input').click()
if (count == 1):
browser.find_by_xpath(fillItemThree).click()
if browser.is_text_present(fillSizeThree):
browser.find_option_by_text(fillSizeThree).first.click()
count = 20
browser.find_by_xpath('//*[@id="add-remove-buttons"]/input').click()
count = count - 1
#click add to cart
#browser.find_by_xpath('//*[@id="add-remove-buttons"]/input').click()
class Session:
def __init__(self, browser, user):
self.browser = Browser(browser)
self.browser.visit('http://jizdenky.studentagency.cz/')
self.browser.fill_form({'passwordAccountCode': user['login'],
'password': user['password']})
self.browser.execute_script('window.scrollTo(0, 100)')
button = self.browser.find_by_value('Přihlásit').first
button.click()
self.user = user
self.log = logging.getLogger(__name__)
def go_search(self):
self.browser.visit('http://jizdenky.studentagency.cz/')
def search(self, task, date_return=None, is_open=False):
self.browser.find_by_id('hp_form_itinerar').first \
.find_by_xpath('div/input[@type="radio"]'
)[1 if date_return or is_open else 0].check()
for city, i in [(task.from_city, 1), (task.to_city, 2)]:
self.browser.find_by_css('input[tabindex="{}"]'.format(i)) \
.first.fill(city)
for item in self.browser.find_by_css('.ui-menu-item'):
link = item.find_by_tag('a')
if link.value.lower() == city.lower():
link.click()
break
self.browser.fill('departure:dateField', task.date)
if date_return:
self.browser.fill('returnDeparture:dateField', date_return)
if is_open:
self.browser.check('returnTicketOpen')
self.browser.find_option_by_text('ISIC').first.check()
self.browser.find_by_value('Vyhledat').first.click()
while self.browser.is_element_not_present_by_css('.left_column',
wait_time=1):
pass
items = self.browser.find_by_css('.left_column') \
.find_by_xpath('div/div/*')
connections = []
for item in items:
if item.tag_name == 'h2':
date_local = item.text.split(' ')[1]
elif item.tag_name == 'div' and item.has_class('routeSummary'):
assert date_local
if date_local != task.date:
break
connections.append(Connection(item))
return connections
def order_time(self, connection):
while True:
if connection.click():
self.browser
dialog = self.browser.find_by_css('[id^=_wicket_window]')
if dialog:
dialog.first.find_by_tag('button').click()
if self.browser.is_element_present_by_id('sumary_lines',
wait_time=1):
break
self.browser.find_by_id('sumary_lines') \
.first.find_by_tag('button') \
.first.click()
seats = {}
bus = self.browser.find_by_css('.seatsContainer')
if bus:
for seat in bus.first.find_by_css(
'.seatContainer:not([style*=blocked])'):
seats[int(seat.find_by_tag('div').first.html[:-1])] = seat
else:
bus = self.browser.find_by_css('.vehicle')
for seat in bus.first.find_by_css('.free, .selected'):
seats[int(seat.text[:-1])] = seat
return seats
def order_seat(self, seat):
if not seat.has_class('selected'):
seat.click()
for fs in self.browser.find_by_css('fieldset.topRoute'):
legend = fs.find_by_css('legend')
if legend and 'Pojištění' in legend[0].text:
for package in fs.find_by_css('.insurancePackageType'):
if 'nechci' in package.find_by_tag('label').text:
package.find_by_tag('input').click()
time.sleep(1)
submit = self.browser.find_by_css('[name^=buttonContainer]').first
interaction_type = submit.text
reserved = 'Rezervovat' in interaction_type
if not reserved:
submit.click()
time.sleep(1)
data = (self.user['first'],
self.user['last'],
self.user['email'],
self.user['phone'])
for item, value in zip(self.browser.find_by_id('passengerInfo')
.first.find_by_tag('input'),
data):
item.fill(value)
submit = self.browser.find_by_css('[name^=buttonContainer]').first
interaction_type = submit.text
assert 'Rezervovat' in interaction_type
agreement = self.browser.find_by_css('[name="bottomComponent:termsAgreementCont:termsAgreementCB"]')
if agreement:
agreement[0].check()
time.sleep(1)
submit.click()
with open('conf.yaml') as f:
conf = yaml.load(f)
if 'email' in conf:
email = conf['email']
while self.browser.is_element_not_present_by_id('ticketPage', wait_time=1):
pass
msg = MIMEText(self.browser.find_by_id('ticketPage').first.html, 'html')
msg['Subject'] = 'SA reservation'
msg['From'] = email['from']
msg['To'] = self.user['email']
username = email['username']
password = email['password']
server = smtplib.SMTP(email['server'])
server.starttls()
server.login(username, b64decode(password).decode())
server.sendmail(msg['From'], msg['To'], msg.as_string())
server.quit()
class supremeBot(object):
path = ""
linksToAvoid = []
def __init__(self, **info):
self.base_url = 'http://www.supremenewyork.com/'
self.shop = 'shop/all/'
self.checkout = 'checkout/'
self.info = info
def initializeBrowser(self):
global path
driver = self.info["driver"]
path = helpers.get_driver_path(driver)
if driver == "geckodriver":
self.b = Browser()
elif driver == "chromedriver":
executable_path = {"executable_path": path}
self.b = Browser('chrome', headless=False, **executable_path)
def findProduct(self):
try:
r = requests.get("{}{}{}".format(self.base_url, self.shop,
self.info['category'])).text
soup = bs.BeautifulSoup(r, 'lxml')
temp_tuple = []
temp_link = []
for link in soup.find_all('a', href=True):
temp_tuple.append((link['href'], link.text))
#print(temp_tuple)
for i in temp_tuple:
if (any(x in i[1] for x in self.info['product'])
and (i[0] in self.linksToAvoid) != True):
r = requests.get("{}{}".format(self.base_url,
str(i[0]))).text
soup = bs.BeautifulSoup(r, 'html.parser')
if ("<b class=\"button sold-out\">sold out</b>"
in str(soup.find_all('b'))):
self.linksToAvoid.append(i[0])
print("Sold out: " + str(i[1]))
else:
temp_link.append(i[0])
if (len(temp_link) != 0 and (i[1] == self.info['color'])
and (i[0] in self.linksToAvoid) != True):
temp_link.append(i[0])
break
elif (len(temp_link) != 0 and ("" == self.info['color'])
and (i[0] in self.linksToAvoid) != True):
temp_link.append(i[0])
break
self.final_link = temp_link[len(temp_link) - 1]
return True
except:
return False
def addToCart(self):
self.b.visit("{}{}".format(self.base_url, str(self.final_link)))
if (self.info['size'] == ""):
#do nothing
None
else:
self.b.find_option_by_text(self.info['size']).click()
try:
self.b.find_by_value('add to cart').click()
try:
time.sleep(.1)
self.checkoutFunc()
except:
time.sleep(.892)
print("trying to add to cart again")
self.addToCart()
except:
print("OOS, Checking diff color")
self.info['color'] = ""
self.linksToAvoid.append(self.final_link)
self.findProduct()
self.addToCart()
def checkoutFunc(self):
self.b.visit("{}{}".format(self.base_url, self.checkout))
self.b.fill("order[billing_name]", self.info['namefield'])
self.b.fill("order[email]", self.info['emailfield'])
self.b.fill("order[tel]", self.info['phonefield'])
self.b.fill("order[billing_address]", self.info['addressfield'])
#self.b.fill("order[billing_city]", self.info['city'])
self.b.fill("order[billing_zip]", self.info['zip'])
self.b.select("order[billing_country]", self.info['country'])
self.b.fill("riearmxa", self.info['number'])
self.b.select("credit_card[month]", self.info['month'])
self.b.select("credit_card[year]", self.info['year'])
self.b.fill("credit_card[meknk]", self.info['ccv'])
self.b.find_by_css('.terms').click()
self.b.find_by_value("process payment").click()
def quitBot(self):
self.b.quit()
class Session:
def __init__(self, browser, user):
self.browser = Browser(browser)
self.browser.visit('http://jizdenky.studentagency.cz/')
self.browser.fill_form({
'passwordAccountCode': user['login'],
'password': user['password']
})
self.browser.execute_script('window.scrollTo(0, 100)')
button = self.browser.find_by_value('Přihlásit').first
button.click()
self.user = user
self.log = logging.getLogger(__name__)
def go_search(self):
self.browser.visit('http://jizdenky.studentagency.cz/')
def search(self, task, date_return=None, is_open=False):
self.browser.find_by_id('hp_form_itinerar').first \
.find_by_xpath('div/input[@type="radio"]'
)[1 if date_return or is_open else 0].check()
for city, i in [(task.from_city, 1), (task.to_city, 2)]:
self.browser.find_by_css('input[tabindex="{}"]'.format(i)) \
.first.fill(city)
for item in self.browser.find_by_css('.ui-menu-item'):
link = item.find_by_tag('a')
if link.value.lower() == city.lower():
link.click()
break
self.browser.fill('departure:dateField', task.date)
if date_return:
self.browser.fill('returnDeparture:dateField', date_return)
if is_open:
self.browser.check('returnTicketOpen')
self.browser.find_option_by_text('ISIC').first.check()
self.browser.find_by_value('Vyhledat').first.click()
while self.browser.is_element_not_present_by_css('.left_column',
wait_time=1):
pass
items = self.browser.find_by_css('.left_column') \
.find_by_xpath('div/div/*')
connections = []
for item in items:
if item.tag_name == 'h2':
date_local = item.text.split(' ')[1]
elif item.tag_name == 'div' and item.has_class('routeSummary'):
assert date_local
if date_local != task.date:
break
connections.append(Connection(item))
return connections
def order_time(self, connection):
while True:
if connection.click():
self.browser
dialog = self.browser.find_by_css('[id^=_wicket_window]')
if dialog:
dialog.first.find_by_tag('button').click()
if self.browser.is_element_present_by_id('sumary_lines',
wait_time=1):
break
self.browser.find_by_id('sumary_lines') \
.first.find_by_tag('button') \
.first.click()
seats = {}
bus = self.browser.find_by_css('.seatsContainer')
if bus:
for seat in bus.first.find_by_css(
'.seatContainer:not([style*=blocked])'):
seats[int(seat.find_by_tag('div').first.html[:-1])] = seat
else:
bus = self.browser.find_by_css('.vehicle')
for seat in bus.first.find_by_css('.free, .selected'):
seats[int(seat.text[:-1])] = seat
return seats
def order_seat(self, seat):
if not seat.has_class('selected'):
seat.click()
for fs in self.browser.find_by_css('fieldset.topRoute'):
legend = fs.find_by_css('legend')
if legend and 'Pojištění' in legend[0].text:
for package in fs.find_by_css('.insurancePackageType'):
if 'nechci' in package.find_by_tag('label').text:
package.find_by_tag('input').click()
time.sleep(1)
submit = self.browser.find_by_css('[name^=buttonContainer]').first
interaction_type = submit.text
reserved = 'Rezervovat' in interaction_type
if not reserved:
submit.click()
time.sleep(1)
data = (self.user['first'], self.user['last'], self.user['email'],
self.user['phone'])
for item, value in zip(
self.browser.find_by_id('passengerInfo').first.find_by_tag(
'input'), data):
item.fill(value)
submit = self.browser.find_by_css('[name^=buttonContainer]').first
interaction_type = submit.text
assert 'Rezervovat' in interaction_type
agreement = self.browser.find_by_css(
'[name="bottomComponent:termsAgreementCont:termsAgreementCB"]')
if agreement:
agreement[0].check()
time.sleep(1)
submit.click()
with open('conf.yaml') as f:
conf = yaml.load(f)
if 'email' in conf:
email = conf['email']
while self.browser.is_element_not_present_by_id('ticketPage',
wait_time=1):
pass
msg = MIMEText(
self.browser.find_by_id('ticketPage').first.html, 'html')
msg['Subject'] = 'SA reservation'
msg['From'] = email['from']
msg['To'] = self.user['email']
username = email['username']
password = email['password']
server = smtplib.SMTP(email['server'])
server.starttls()
server.login(username, b64decode(password).decode())
server.sendmail(msg['From'], msg['To'], msg.as_string())
server.quit()
