def __init__(self,
urn=None,
pwd=None,
init_dict=[res for res in permutation(lowercase, 2)]):
if not os.path.exists(self.raw_file_folder):
os.mkdir(self.raw_file_folder)
os.chdir(self.raw_file_folder)
if not urn:
urn = raw_input('Enter the username: '******''
else:
print 'Username: '******'\n'
if not pwd:
pwd = getpass.getpass('Enter the password: '******'cls')
with open(self.err_log_name, 'w') as f:
f.write('Starting crawling from %s...\n' % time.ctime())
self.url = 'https://aisapps.vassar.edu/directory/internal_auth_studir_srch.php'
print 'Initializing...\n'
self.browser = PhantomJS()
self.i = 0
self.urn = urn
self.pwd = pwd
self.DICT = init_dict
self.searchReq = 'https://aisapps.vassar.edu/directory/internal_auth_studirctry_rslts.php'
self.init()
def get_phantom():
caps = DesiredCapabilities.PHANTOMJS
caps["phantomjs.page.settings.userAgent"] = USER_AGENT
caps["pageLoadStrategy"] = "eager"
browser = PhantomJS(desired_capabilities=caps,
service_args=['--ignore-ssl-errors=true'])
browser.set_page_load_timeout(5)
WebDriverWait(browser, 10).until(lambda driver: driver.execute_script(
'return document.readyState') != 'loading')
return browser
def login(self, url):
self.driver = PhantomJS(executable_path=self.executable_path)
self.driver.get(url)
self.driver.find_element_by_id('userID').send_keys(self.username)
self.driver.find_element_by_id('password').send_keys(self.password)
self.driver.find_element_by_xpath("//input[@value='Log in']").click()
WebDriverWait(self.driver, self.timeout).until(
EC.text_to_be_present_in_element((By.XPATH, "//a[@alt='logout']"),
'Log Out'),
'%s failed to login within %d seconds timeout' %
(self.username, self.timeout))
class Factory:
def __init__(self,
username: str,
password: str,
timeout: int = 30,
executable_path: str = None):
self.username = username
self.password = password
self.timeout = timeout
self.executable_path = executable_path if executable_path else 'phantomjs'
def login(self, url):
self.driver = PhantomJS(executable_path=self.executable_path)
self.driver.get(url)
self.driver.find_element_by_id('userID').send_keys(self.username)
self.driver.find_element_by_id('password').send_keys(self.password)
self.driver.find_element_by_xpath("//input[@value='Log in']").click()
WebDriverWait(self.driver, self.timeout).until(
EC.text_to_be_present_in_element((By.XPATH, "//a[@alt='logout']"),
'Log Out'),
'%s failed to login within %d seconds timeout' %
(self.username, self.timeout))
def start(self, method: Callable):
method()
WebDriverWait(self.driver, self.timeout).until(
is_page_read(),
'Opertation failed to be completed before the %d seconds timeout' %
self.timeout)
def get_phantomjs_driver_with_proxy(self, proxy_info, scheme='http'):
webdriver.DesiredCapabilities.PHANTOMJS[
'phantomjs.page.settings.userAgent'] = self.ua
webdriver.DesiredCapabilities.PHANTOMJS[
'phantomjs.page.settings.resourceTimeout'] = '5000'
if 'proxy_username' in proxy_info and 'proxy_password' in proxy_info:
service_args = [
'--proxy=%s:%s' %
(proxy_info['proxy_host'], proxy_info['proxy_port']),
'--proxy-auth=%s:%s' %
(proxy_info['proxy_username'], proxy_info['proxy_password']),
'--proxy-type=%s' % (scheme),
'--ignore-ssl-errors=true',
]
else:
service_args = [
'--proxy=%s:%s' %
(proxy_info['proxy_host'], proxy_info['proxy_port']),
'--proxy-type=%s' % (scheme),
'--ignore-ssl-errors=true',
'--load-images=false',
]
browser = PhantomJS(service_args=service_args)
self.logger.info('set proxy %s:%s source ip: %s type: %d',
proxy_info['proxy_host'],
proxy_info.get('proxy_username'),
proxy_info['source_ip'], proxy_info['type'])
return browser
class DLinkScanner(DeviceScanner):
def __init__(self, gateway, admin_password):
self.password = admin_password
self.gateway = gateway
self.driver = None
self.success_init = True
self.last_results = []
results = self._update_info()
self.success_init = results is not None
@Throttle(MIN_TIME_BETWEEN_SCANS)
def _update_info(self):
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, StaleElementReferenceException
from seleniumrequests import PhantomJS
self.driver = PhantomJS(
'/home/homeassistant/phantomjs-raspberrypi/bin/phantomjs')
path = "http://" + self.gateway + "/info/Login.html"
self.driver.get(path)
delay = 15 # seconds
try:
WebDriverWait(self.driver, delay).until(
EC.presence_of_element_located((By.ID, "admin_Password")))
_LOGGER.info('Logging in')
self.driver.find_element_by_id('admin_Password').send_keys(
self.password)
self.driver.find_element_by_id('admin_Password').send_keys(
Keys.RETURN)
self.driver.find_element_by_id("logIn_btn").click()
WebDriverWait(self.driver, delay).until(
EC.element_to_be_clickable((By.ID, "clientInfo_circle")))
_LOGGER.info('Navigating to client list')
self.driver.find_element_by_id("clientInfo_circle").click()
# Bad timing, refresh elements
attempts = 0
success = False
while success is False and attempts < 4:
try:
WebDriverWait(self.driver, delay).until(
EC.presence_of_element_located(
(By.CLASS_NAME, "client_Name")))
elements = self.driver.find_elements_by_class_name(
'client_Name')
clients = []
for val in elements:
_LOGGER.info('Found ' + str(val.text))
clients.extend([str(val.text)])
success = True
except StaleElementReferenceException:
attempts += 1
except TimeoutException:
_LOGGER.error('Timeout exception')
self.driver.save_screenshot('error.png')
clients = []
self.driver.service.process.send_signal(
signal.SIGTERM) # kill the specific phantomjs child proc
self.driver.quit() # quit the node proc
self.last_results = clients
return clients
def is_client_connected(self, client_name):
clients = self._update_info()
if clients is not None:
return client_name in clients
else:
return False
def get_device_name(self, device):
"""The firmware doesn't save the name of the wireless device."""
return None
def scan_devices(self):
"""Scan for new devices and return a list with found device Namess."""
self._update_info()
return self.last_results
def write_article(date, body):
url = 'http://gall.dcinside.com/mgallery/board/write/?id=nendoroid'
xpaths = {
'nick': "//input[@name='name']",
'pw': "//input[@name='password']",
'title': "//input[@name='subject']",
'body': "//div[@id='tx_canvas_source_holder']",
}
dcap = dict(dc.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 "
"(KHTML, like Gecko) Chrome/16.0.82")
title_format = '[알림] 굿스마샵 {} 넨도 예약 마감 D-{}'
date = date.replace(hour=12)
today = datetime.now(
) #.replace(hour=0, minute=0, second=0, microsecond=0)
dday = date - today
if dday.days > 3 or dday.days < 0:
return
title = title_format.format(date.strftime('%m월%d일'), dday.days)
print(title)
driver = PhantomJS(desired_capabilities=dcap)
driver.get(url)
driver.find_element_by_xpath(xpaths['nick']).send_keys('넨갤봇')
driver.find_element_by_xpath(xpaths['pw']).send_keys('2648')
driver.find_element_by_xpath(xpaths['title']).send_keys(title)
WebDriverWait(driver, 2)
html = driver.find_element_by_xpath("//div[@id='tx_switchertoggle']")
bd = driver.find_element_by_xpath(xpaths['body'])
def make_body(date, body):
b = '<p>{} 정오에 예약이 마감됩니다.</p><br/><p>클릭하면 제품 페이지로 이동합니다.</p><br/><ul>'.format(
date.strftime('%m월 %d일'))
for name, link in body:
b += ('<li><a href="{}" target="_blank">{}</a></li>'.format(
link, name))
b += '</ul>'
return b
full_body = make_body(date, body)
print(full_body)
actions = webdriver.ActionChains(driver)
actions.move_to_element(html)
actions.click()
actions.move_to_element(bd)
actions.click()
actions.pause(1)
actions.send_keys(full_body)
actions.pause(1)
actions.perform()
submit = driver.find_element_by_xpath("//p[@class='btn_box_right']//input")
submit.click()
WebDriverWait(driver, 1)
#print(driver.get_log('browser'))
#print(driver.get_log('har'))
driver.save_screenshot('a.png')
driver.close()
print('done!')
def create_account(username, password, email, birthday, captchakey2, captchatimeout):
if password is not None:
_validate_password(password)
print("Attempting to create user {user}:{pw}. Opening browser...".format(user=username, pw=password))
if captchakey2 != None:
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = user_agent
#driver = webdriver.PhantomJS(desired_capabilities=dcap)
driver = PhantomJS(desired_capabilities=dcap)
else:
#driver = webdriver.Chrome()
driver = Chrome()
driver.set_window_size(600, 600)
# Input age: 1992-01-08
print("Step 1: Verifying age using birthday: {}".format(birthday))
driver.get("{}/sign-up/".format(BASE_URL))
assert driver.current_url == "{}/sign-up/".format(BASE_URL)
elem = driver.find_element_by_name("dob")
# Workaround for different region not having the same input type
driver.execute_script("var input = document.createElement('input'); input.type='text'; input.setAttribute('name', 'dob'); arguments[0].parentNode.replaceChild(input, arguments[0])", elem)
elem = driver.find_element_by_name("dob")
elem.send_keys(birthday)
elem.submit()
# Todo: ensure valid birthday
# Create account page
print("Step 2: Entering account details")
assert driver.current_url == "{}/parents/sign-up".format(BASE_URL)
user = driver.find_element_by_name("username")
user.clear()
user.send_keys(username)
_validate_username(driver, username)
elem = driver.find_element_by_name("password")
elem.clear()
elem.send_keys(password)
elem = driver.find_element_by_name("confirm_password")
elem.clear()
elem.send_keys(password)
elem = driver.find_element_by_name("email")
elem.clear()
elem.send_keys(email)
elem = driver.find_element_by_name("confirm_email")
elem.clear()
elem.send_keys(email)
driver.find_element_by_id("id_public_profile_opt_in_1").click()
driver.find_element_by_name("terms").click()
if captchakey2 == None:
#Do manual captcha entry
print("You did not pass a 2captcha key. Please solve the captcha manually.")
elem = driver.find_element_by_class_name("g-recaptcha")
driver.execute_script("arguments[0].scrollIntoView(true);", elem)
# Waits 1 minute for you to input captcha
try:
WebDriverWait(driver, 60).until(EC.text_to_be_present_in_element_value((By.NAME, "g-recaptcha-response"), ""))
print("Captcha successful. Sleeping for 1 second...")
time.sleep(1)
except TimeoutException, err:
print("Timed out while manually solving captcha")
def mainprocess(captchakey, saveloc):
user_agent = (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36"
)
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = user_agent
driver = PhantomJS(desired_capabilities=dcap,
service_args=['--load-images=no'])
#driver = webdriver.PhantomJS()
driver.set_window_size(1600, 1200)
driver.implicitly_wait(10)
driver.get("https://club.pokemon.com/us/pokemon-trainer-club")
driver.find_element_by_id("user-account-signup").click()
delay = 5 # seconds
try:
myElem = WebDriverWait(driver, delay).until(
EC.presence_of_element_located((By.ID, 'id_dob')))
except TimeoutException:
logging.info("Loading took too much time!")
elem = driver.find_element_by_name("dob")
driver.execute_script(
"var input = document.createElement('input'); input.type='text'; input.setAttribute('name', 'dob'); arguments[0].parentNode.replaceChild(input, arguments[0])",
elem)
randdate = datetime.date(randint(1975, 1990), randint(1, 12),
randint(1, 28))
elem = driver.find_element_by_name("dob")
elem.send_keys(datetime.datetime.strftime(randdate, '%Y-%m-%d'))
driver.save_screenshot('bday4.png')
time.sleep(1)
delay = 5 # seconds
try:
myElem = WebDriverWait(driver, delay).until(
EC.presence_of_element_located((
By.XPATH,
'//*[@id="sign-up-theme"]/section/div/div/div[1]/form/div[2]/div/div/label'
)))
except TimeoutException:
logging.info("Loading took too much time!")
myElem.click()
time.sleep(1)
delay = 5 # seconds
try:
myElem = WebDriverWait(driver, delay).until(
EC.presence_of_element_located((
By.XPATH,
'//*[@id="sign-up-theme"]/section/div/div/div[1]/form/div[2]/div/div/div/div/ul/li[168]'
)))
except TimeoutException:
logging.info("Loading took too much time!")
myElem.click()
driver.find_element_by_xpath(
'//*[@id="sign-up-theme"]/section/div/div/div[1]/form/input[2]').click(
)
randomemail = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
randompass = ''.join(
random.SystemRandom().choice(string.ascii_lowercase +
string.ascii_uppercase + string.digits +
string.punctuation) for _ in range(10))
driver2 = PhantomJS(desired_capabilities=dcap,
service_args=['--load-images=no'])
driver2.set_window_size(1600, 1200)
driver2.get("https://temp-mail.org/en/option/change/")
driver2.find_element_by_name("mail").send_keys(randomemail)
doms = Select(driver2.find_element_by_xpath('//*[@id="domain"]'))
x = doms.options
domain = str(x[0].text)
time.sleep(3)
driver2.find_element_by_id('postbut').click()
driver2.get("https://temp-mail.org/en/option/refresh/")
driver.save_screenshot('screenshot.png')
driver.find_element_by_id('id_username').send_keys(randomemail)
driver.find_element_by_id('check-availability-username').click()
delay = 5 # seconds
try:
myElem = WebDriverWait(driver, delay).until(
EC.presence_of_element_located(
(By.XPATH, '//*[@id="username-suggestion"]/div/h3')))
if driver.find_element_by_xpath(
'//*[@id="username-suggestion"]/div').find_element_by_tag_name(
'h3').text == 'Username is invalid.':
logging.info('Username already in use exiting')
exit()
except TimeoutException:
logging.info("Loading took too much time!")
driver.find_element_by_id('id_password').send_keys(randompass)
driver.find_element_by_id('id_confirm_password').send_keys(randompass)
driver.find_element_by_id('id_email').send_keys(randomemail + domain)
driver.find_element_by_id('id_confirm_email').send_keys(randomemail +
domain)
driver.find_element_by_id('id_screen_name').send_keys(randomemail)
driver.find_element_by_id('check-availability-screen-name').click()
delay = 5 # seconds
try:
myElem = WebDriverWait(driver, delay).until(
EC.presence_of_element_located((
By.XPATH,
'//*[@id="user-signup-create-account-form"]/fieldset[1]/div/div/div[1]/div/div[11]/div/div/h3'
)))
if driver.find_element_by_xpath(
'//*[@id="user-signup-create-account-form"]/fieldset[1]/div/div/div[1]/div/div[11]/div/div'
).find_element_by_tag_name(
'h3'
).text == 'This screen name already exists. Please try the following:':
logging.info('Screen Name already in use exiting')
exit()
except TimeoutException:
logging.info("Loading took too much time!")
driver.find_element_by_id('id_terms').click()
logging.info("Starting captcha solve")
# AutoCapcha
autocaptcha(captchakey, driver)
driver.find_element_by_xpath(
'//*[@id="user-signup-create-account-form"]/fieldset[2]/div/input'
).click()
driver.close()
logging.info("Waiting on Email")
for z in range(0, 10):
try:
delay = 60 # seconds
try:
myElem = WebDriverWait(driver2, delay).until(
EC.presence_of_element_located(
(By.XPATH, '//*[@id="mails"]/tbody')))
except TimeoutException:
if z == 9:
logging.info("Can't Find the email sorries :(")
exit()
else:
logging.info("No Email Yet " + str(z + 1) + "/10")
driver2.refresh()
except:
continue
myElem.click()
time.sleep(5)
elems = driver2.find_elements_by_tag_name('a')
time.sleep(5)
for elem in elems:
test = str(elem.get_attribute("href"))
if "https://club.pokemon.com/us/pokemon-trainer-club/activated/" in test:
actions = ActionChains(driver2)
actions.move_to_element(elem).perform()
elem.click()
break
time.sleep(10)
driver2.quit()
logging.info("Account created saving details")
with open(os.path.join(saveloc, 'accounts.txt'), "a") as myfile:
myfile.write("\n" + randomemail + ' - ' + randompass)
logging.info(randomemail + " account created")
def create_account(username, password, email, birthday, captchakey2,
captchatimeout):
if password:
_validate_password(password)
print("Attempting to create user {user}:{pw}. Opening browser...".format(
user=username, pw=password))
if captchakey2:
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap[
"phantomjs.page.settings.userAgent"] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'
driver = PhantomJS(desired_capabilities=dcap)
else:
driver = Chrome()
driver.set_window_size(600, 600)
# Input age: 1992-01-08
print("Step 1: Verifying age using birthday:", birthday)
driver.get("{}/sign-up/".format(BASE_URL))
assert driver.current_url == "{}/sign-up/".format(BASE_URL)
elem = driver.find_element_by_name("dob")
# Workaround for different region not having the same input type
driver.execute_script(
"var input = document.createElement('input'); input.type='text'; input.setAttribute('name', 'dob'); arguments[0].parentNode.replaceChild(input, arguments[0])",
elem)
elem = driver.find_element_by_name("dob")
elem.send_keys(birthday)
elem.submit()
# Create account page
print("Step 2: Entering account details")
assert driver.current_url == "{}/parents/sign-up".format(BASE_URL)
user = driver.find_element_by_name("username")
user.clear()
user.send_keys(username)
_validate_username(driver, username)
elem = driver.find_element_by_name("password")
elem.clear()
elem.send_keys(password)
elem = driver.find_element_by_name("confirm_password")
elem.clear()
elem.send_keys(password)
elem = driver.find_element_by_name("email")
elem.clear()
elem.send_keys(email)
elem = driver.find_element_by_name("confirm_email")
elem.clear()
elem.send_keys(email)
driver.find_element_by_id("id_public_profile_opt_in_1").click()
driver.find_element_by_name("terms").click()
if not captchakey2:
#Do manual captcha entry
print(
"You did not pass a 2captcha key. Please solve the captcha manually."
)
elem = driver.find_element_by_class_name("g-recaptcha")
driver.execute_script("arguments[0].scrollIntoView(true);", elem)
# Waits 90 seconds for you to input captcha
try:
WebDriverWait(driver, 90).until(
EC.text_to_be_present_in_element_value(
(By.NAME, "g-recaptcha-response"), ""))
print("Captcha successful.")
except TimeoutException:
print("Timed out while manually solving captcha")
else:
# Now to automatically handle captcha
print("Starting autosolve recaptcha")
html_source = driver.page_source
gkey_index = html_source.find(
"https://www.google.com/recaptcha/api2/anchor?k=") + 47
gkey = html_source[gkey_index:gkey_index + 40]
recaptcharesponse = "Failed"
while (recaptcharesponse == "Failed"):
recaptcharesponse = openurl("http://2captcha.com/in.php?key=" +
captchakey2 +
"&method=userrecaptcha&googlekey=" +
gkey + "&pageurl=" +
driver.current_url)
captchaid = recaptcharesponse[3:]
recaptcharesponse = "CAPCHA_NOT_READY"
elem = driver.find_element_by_class_name("g-recaptcha")
print("We will wait 10 seconds for captcha to be solved by 2captcha")
start_time = time.monotonic()
timedout = False
while recaptcharesponse == "CAPCHA_NOT_READY":
time.sleep(10)
elapsedtime = time.monotonic() - start_time
if elapsedtime > captchatimeout:
print("Captcha timeout reached. Exiting.")
timedout = True
break
print("Captcha still not solved, waiting another 10 seconds.")
recaptcharesponse = "Failed"
while (recaptcharesponse == "Failed"):
recaptcharesponse = openurl(
"http://2captcha.com/res.php?key={}&action=get&id={}".
format(captchakey2, captchaid))
if timedout == False:
solvedcaptcha = recaptcharesponse[3:]
elem = driver.find_element_by_name("g-recaptcha-response")
elem = driver.execute_script(
"arguments[0].style.display = 'block'; return arguments[0];",
elem)
elem.send_keys(solvedcaptcha)
print("Solved captcha")
try:
user.submit()
except StaleElementReferenceException:
print("Error StaleElementReferenceException!")
try:
_validate_response(driver)
except Exception:
print("Failed to create user:"******"Account successfully created.")
driver.quit()
return True
def delete(self, user):
''' Delete the user on Github Enterprise '''
# Initialize the PhantomJS selenium driver
driver = PhantomJS()
driver.implicitly_wait(10)
driver.set_window_size(1400, 850)
# Login as the admin user
driver.get('https://%s/login' % (self.ghe_host))
driver.find_element_by_name('login').send_keys(self.ghe_user)
driver.find_element_by_name('password').send_keys(self.ghe_pass)
driver.find_element_by_name('commit').click()
# Check for two-factor auth code request
if driver.current_url == 'https://%s/sessions/two-factor' % self.ghe_host:
if self.ghe_totp:
base = '.auth-form-body input'
u = driver.find_element_by_css_selector('%s[name=utf8]' % base)
t = driver.find_element_by_css_selector('%s[name=authenticity_token]' % base)
otp = pyotp.TOTP(self.ghe_totp)
driver.request('POST', 'https://%s/sessions/two-factor' % self.ghe_host,
data={
'utf8': u.get_attribute('value'),
'otp': otp.now(),
'authenticity_token': t.get_attribute('value')
}
)
else:
print('Two-Factor authentication required.')
sys.exit()
# Retrieve the admin page for the designated user to be deleted
driver.get('https://%s/stafftools/users/%s/admin' % (self.ghe_host, user))
# Ensure that we were able to access the requested admin page
if 'Page not found' in driver.title or user.lower() not in driver.title.lower():
print('User not found, or insufficient access rights.')
sys.exit()
# Locate the necessary inputs to be able to delete a user
base = '#confirm_deletion form input'
u = driver.find_element_by_css_selector('%s[name=utf8]' % base)
m = driver.find_element_by_css_selector('%s[name=_method]' % base)
t = driver.find_element_by_css_selector('%s[name=authenticity_token]' % base)
# Send the delete user request
driver.request('POST', 'https://%s/stafftools/users/%s' % (self.ghe_host, user),
data={
'utf8': u.get_attribute('value'),
'_method': m.get_attribute('value'),
'authenticity_token': t.get_attribute('value')
}
)
def update(self, user, email):
''' Reset the users email address on Github Enterprise '''
# Initialize the PhantomJS selenium driver
driver = PhantomJS()
driver.implicitly_wait(10)
driver.set_window_size(1400, 850)
# Login as the admin user
driver.get('https://%s/login' % (self.ghe_host))
driver.find_element_by_name('login').send_keys(self.ghe_user)
driver.find_element_by_name('password').send_keys(self.ghe_pass)
driver.find_element_by_name('commit').click()
# Check for two-factor auth code request
if driver.current_url == 'https://%s/sessions/two-factor' % self.ghe_host:
if self.ghe_totp:
base = '.auth-form-body input'
u = driver.find_element_by_css_selector('%s[name=utf8]' % base)
t = driver.find_element_by_css_selector(
'%s[name=authenticity_token]' % base)
otp = pyotp.TOTP(self.ghe_totp)
driver.request(
'POST',
'https://%s/sessions/two-factor' % self.ghe_host,
data={
'utf8': u.get_attribute('value'),
'otp': otp.now(),
'authenticity_token': t.get_attribute('value')
})
else:
print('Two-Factor authentication required.')
sys.exit()
# Retrieve the email admin page for the designated user to be updated
driver.get('https://%s/stafftools/users/%s/emails' %
(self.ghe_host, user))
# Ensure that we were able to access the requested admin page
if 'Page not found' in driver.title or user.lower(
) not in driver.title.lower():
print('User not found, or insufficient access rights.')
sys.exit()
# Locate the necessary inputs to be able to add an email address
base = 'form[action="/stafftools/users/%s/emails"] input' % user
u = driver.find_element_by_css_selector('%s[name=utf8]' % base)
t = driver.find_element_by_css_selector('%s[name=authenticity_token]' %
base)
# Send the add email address request
driver.request('POST',
'https://%s/stafftools/users/%s/emails' %
(self.ghe_host, user),
data={
'utf8': u.get_attribute('value'),
'email': email,
'authenticity_token': t.get_attribute('value')
})
# Send password reset to new email address
base = 'form[action="/stafftools/users/%s/password/send_reset_email"] input' % user
u = driver.find_element_by_css_selector('%s[name=utf8]' % base)
t = driver.find_element_by_css_selector('%s[name=authenticity_token]' %
base)
m = driver.find_element_by_css_selector('%s[name=_method]' % base)
driver.request(
'POST',
'https://%s/stafftools/users/%s/password/send_reset_email' %
(self.ghe_host, user),
data={
'utf8': u.get_attribute('value'),
'email': email,
'authenticity_token': t.get_attribute('value'),
'_method': m.get_attribute('value')
})
# Get password reset link and display to console
driver.get('https://%s/stafftools/users/%s/emails' %
(self.ghe_host, user))
if email in driver.page_source:
print('Email added and password reset email sent.')
else:
print(
'New email not showing up on user page; please check manually.'
)
def webdriver_class():
webdriver = PhantomJS()
# Fix Selenium PermissionError when trying to delete PhantomJS cookie files by simply not creating a cookie
# file.
webdriver.service._cookie_temp_file = None
return webdriver
def __init__(self,
web_browser='Chrome',
webdriver_path='/Users/sebas12/Downloads/chromedriver',
*args,
**kwargs):
# instantiate the web browser that will be used in this function.
if web_browser.lower() == 'chrome':
self.driver = Chrome(webdriver_path)
elif web_browser.lower() == 'firefox':
self.driver = Firefox(webdriver_path)
elif web_browser.lower() == 'opera':
self.driver == Opera(webdriver_path)
elif web_browser.lower() == 'phantomjs':
self.driver = PhantomJS(webdriver_path)
elif web_browser.lower() == 'safari':
self.driver = Safari(webdriver_path)
self.driver.set_script_timeout(15)
self.base_url = kwargs.get('base_url', 'https://stats.nba.com/stats/')
# for future convinence.
self.last_url_called = None
# for future inspection
self.API_params = None
self.teamIDS_dict = {
'ATL': '1610612737',
'BOS': '1610612738',
'BKN': '1610612751',
'CHA': '1610612766',
'CHI': '1610612741',
'CLE': '1610612739',
'DAL': '1610612742',
'DEN': '1610612743',
'DET': '1610612765',
'GSW': '1610612744',
'HOU': '1610612745',
'IND': '1610612754',
'LAC': '1610612746',
'LAL': '1610612747',
'MEM': '1610612763',
'MIA': '1610612748',
'MIL': '1610612749',
'MIN': '1610612750',
'NOP': '1610612740',
'NYK': '1610612752',
'OKC': '1610612760',
'ORL': '1610612753',
'PHL': '1610612755',
'PHX': '1610612756',
'POR': '1610612757',
'SAC': '1610612758',
'SAS': '1610612759',
'TOR': '1610612761',
'UTA': '1610612762',
'WAS': '1610612764'
}
self.supported_endpoints = [
'boxscoreadvancedv2', 'boxscorefourfactorsv2', 'boxscoremiscv2',
'boxscorescoringv2', 'boxscoretraditionalv2', 'boxscoreusagev2'
]
self.headers_to_keep_dict = {
'boxscoreadvancedv2': [
"TEAM_ID", "PLAYER_ID", "MIN", "E_OFF_RATING", "OFF_RATING",
"E_DEF_RATING", "DEF_RATING", "E_NET_RATING", "NET_RATING",
"AST_PCT", "AST_TOV", "AST_RATIO", "OREB_PCT", "DREB_PCT",
"REB_PCT", "TM_TOV_PCT", "EFG_PCT", "TS_PCT", "USG_PCT",
"E_USG_PCT", "E_PACE", "PACE", "PIE"
],
'boxscorefourfactorsv2': [
"TEAM_ID", "PLAYER_ID", "FTA_RATE", "OPP_EFG_PCT",
"OPP_FTA_RATE", "OPP_TOV_PCT", "OPP_OREB_PCT"
],
'boxscoremiscv2': [
"TEAM_ID", "PLAYER_ID", "PTS_OFF_TOV", "PTS_2ND_CHANCE",
"PTS_FB", "PTS_PAINT", "OPP_PTS_OFF_TOV", "OPP_PTS_2ND_CHANCE",
"OPP_PTS_FB", "OPP_PTS_PAINT", "BLK", "BLKA", "PF", "PFD"
],
'boxscorescoringv2': [
"TEAM_ID", "PLAYER_ID", "PCT_FGA_2PT", "PCT_FGA_3PT",
"PCT_PTS_2PT", "PCT_PTS_2PT_MR", "PCT_PTS_3PT", "PCT_PTS_FB",
"PCT_PTS_FT", "PCT_PTS_OFF_TOV", "PCT_PTS_PAINT",
"PCT_AST_2PM", "PCT_UAST_2PM", "PCT_AST_3PM", "PCT_UAST_3PM",
"PCT_AST_FGM", "PCT_UAST_FGM"
],
'boxscoretraditionalv2': [
"TEAM_ID", "PLAYER_ID", "FGM", "FGA", "FG_PCT", "FG3M", "FG3A",
"FG3_PCT", "FTM", "FTA", "FT_PCT", "OREB", "DREB", "REB",
"AST", "STL", "TO", "PTS", "PLUS_MINUS"
],
'boxscoreusagev2': [
"TEAM_ID", "PLAYER_ID", "PCT_FGM", "PCT_FGA", "PCT_FG3M",
"PCT_FG3A", "PCT_FTM", "PCT_FTA", "PCT_OREB", "PCT_DREB",
"PCT_REB", "PCT_AST", "PCT_TOV", "PCT_STL", "PCT_BLK",
"PCT_BLKA", "PCT_PF", "PCT_PFD", "PCT_PTS"
]
}
pass
def _update_info(self):
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, StaleElementReferenceException
from seleniumrequests import PhantomJS
self.driver = PhantomJS(
'/home/homeassistant/phantomjs-raspberrypi/bin/phantomjs')
path = "http://" + self.gateway + "/info/Login.html"
self.driver.get(path)
delay = 15 # seconds
try:
WebDriverWait(self.driver, delay).until(
EC.presence_of_element_located((By.ID, "admin_Password")))
_LOGGER.info('Logging in')
self.driver.find_element_by_id('admin_Password').send_keys(
self.password)
self.driver.find_element_by_id('admin_Password').send_keys(
Keys.RETURN)
self.driver.find_element_by_id("logIn_btn").click()
WebDriverWait(self.driver, delay).until(
EC.element_to_be_clickable((By.ID, "clientInfo_circle")))
_LOGGER.info('Navigating to client list')
self.driver.find_element_by_id("clientInfo_circle").click()
# Bad timing, refresh elements
attempts = 0
success = False
while success is False and attempts < 4:
try:
WebDriverWait(self.driver, delay).until(
EC.presence_of_element_located(
(By.CLASS_NAME, "client_Name")))
elements = self.driver.find_elements_by_class_name(
'client_Name')
clients = []
for val in elements:
_LOGGER.info('Found ' + str(val.text))
clients.extend([str(val.text)])
success = True
except StaleElementReferenceException:
attempts += 1
except TimeoutException:
_LOGGER.error('Timeout exception')
self.driver.save_screenshot('error.png')
clients = []
self.driver.service.process.send_signal(
signal.SIGTERM) # kill the specific phantomjs child proc
self.driver.quit() # quit the node proc
self.last_results = clients
return clients
driver.find_elements_by_id('s2id_autogen2_search')[0].send_keys('California')
driver.find_elements_by_id('s2id_autogen2_search')[0].send_keys(Keys.RETURN)
###driver.find_elements_by_css_selector('.tabCompTR .tabCompTD .divCompList')[0].click()
##raw_input("Press Enter to continue...")
thankyou = driver.find_elements_by_css_selector('.landing-page .content-channel h2')[0].text
print thankyou
except Exception as exception:
print type(exception).__name__ , 'input error ' , checkingURL
return
driver = webdriver.Firefox()
##driver = webdriver.PhantomJS()
try:
subdriver = PhantomJS()
while True:
if checkform in formUrls:
formURL = formUrls[checkform]
else:
formURL = checkform
if 'resources' in formURL:
checkform = 'contact'
elif 'server-suite-form' in formURL or 'mac-form' in formURL:
checkform = 'server'
elif 'privilege-service-form' in formURL or 'identity-service-form' in formURL:
checkform = 'identity'
elif 'solutions' in formURL:
from seleniumrequests import PhantomJS
from time import sleep
driver = PhantomJS()
driver.set_window_size(1120, 550)
driver.get("https://duckduckgo.com/")
driver.find_element_by_id('search_form_input_homepage').send_keys("realpython")
driver.find_element_by_id("search_button_homepage").click()
print(driver.current_url)
driver.quit()
username = os.environ['DATCU_USERNAME']
password = os.environ['DATCU_PASSWORD']
import json
challenges = None
with open('challenges.json', 'r') as fo:
challenges = json.load(fo)
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from seleniumrequests import PhantomJS
p = PhantomJS(service_log_path=os.path.devnull)
p.set_window_size(1440, 900)
p.get('https://www.mydatcu.org/User/AccessSignin/Start')
p.find_element_by_name('UsernameField').send_keys(username + Keys.RETURN)
p.find_element_by_name('PasswordField').send_keys(password + Keys.RETURN)
# NOTE: This will obviously break if the site ever changes layout
challenge_xpath = "id('AccessForm')/div/div[1]/div[2]/table/tbody/tr[2]/td[2]"
while p.current_url == 'https://www.mydatcu.org/User/AccessSignin/Challenge':
text = p.find_elements_by_xpath(challenge_xpath)
if not text:
raise Exception('No results found!')
text = text[0].text.replace('explain', '').strip()
if text not in challenges:
class Vassar_Downloader(object):
err_log_name = 'ERROR_LOG.txt'
download_log = "temp.txt"
raw_file_folder = 'Raw files -- %s' % str(time.localtime()[0:3])
def __init__(self,
urn=None,
pwd=None,
init_dict=[res for res in permutation(lowercase, 2)]):
if not os.path.exists(self.raw_file_folder):
os.mkdir(self.raw_file_folder)
os.chdir(self.raw_file_folder)
if not urn:
urn = raw_input('Enter the username: '******''
else:
print 'Username: '******'\n'
if not pwd:
pwd = getpass.getpass('Enter the password: '******'cls')
with open(self.err_log_name, 'w') as f:
f.write('Starting crawling from %s...\n' % time.ctime())
self.url = 'https://aisapps.vassar.edu/directory/internal_auth_studir_srch.php'
print 'Initializing...\n'
self.browser = PhantomJS()
self.i = 0
self.urn = urn
self.pwd = pwd
self.DICT = init_dict
self.searchReq = 'https://aisapps.vassar.edu/directory/internal_auth_studirctry_rslts.php'
self.init()
def init(self):
self.if_continue()
self.cluster()
def if_continue(self):
self.finished_list = []
if os.path.exists(self.download_log):
with open(self.download_log, 'r') as f:
data = f.read()
self.finished_list = data.split(',')
self.finished_list.pop(-2)
self.waiting_list = filter(
lambda x: x not in self.finished_list, self.DICT)
self.status = "Continue from the last dumping... %d / %d\n" % (
len(self.DICT) - len(self.waiting_list), len(self.DICT))
print self.status
else:
self.waiting_list = self.DICT
def login(self):
print 'Connecting to "%s"...' % self.url
self.browser.get(self.url)
self.status = '\nTrying to login as %s...' % self.urn
print self.status
username = self.browser.find_element_by_name("username")
password = self.browser.find_element_by_name("password")
username.send_keys(self.urn)
password.send_keys(self.pwd)
login_attempt = self.browser.find_element_by_xpath(
"//*[@type='submit']")
login_attempt.submit()
return self.browser.page_source
def request(self, data, method='POST', error_time=0):
filename = 'source - %s.txt' % (str(data.values()))
#print 'downloading %s...'%filename
try:
resp = self.browser.request(method, self.searchReq, data=data)
with open(filename, 'w') as f:
#f.write(clean_content(resp.content))
f.write(resp.content)
temp = data['lastname']
self.finished_list.append(temp)
self.waiting_list.remove(temp)
self.status = '%d / %d done...' % (
self.total - len(self.waiting_list), self.total)
print self.status
with open(self.download_log, 'a') as f:
f.write('%s,' % temp)
except Exception as e:
error_time += 1
if error_time <= 3:
self.logger('Error! %s failed %d times.' %
(str(data), error_time))
self.request(data, error_time=error_time)
else:
self.logger('<<Error! %s not downloaded!>>' % (str(data)))
def cluster(self):
s = self.login()
if 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd' not in s:
self.status = '\nAuthentification failed.\n'
print self.status
self.logger('Authentification failed.')
return None
else:
self.status = '\nAuthentification succeeded.\n\nDownloading data...\n\n'
print self.status
self.total = len(self.DICT)
threads = [
MyThread(self.request, ({
'lastname': ln
}, 'POST'), name=ln) for ln in self.waiting_list
]
for t in threads:
t.start()
for t in threads:
t.join()
self.status = '\nClearing cache...\n'
print self.status
os.remove(self.download_log)
print 'Done...\n'
def logger(self, msg):
with open(self.err_log_name, 'a') as f:
f.write('%s --- %s\n' % (msg, time.ctime()))