Пример #1
0
def login(url, q, p):
    browser = Browser('chrome')
    #browser = Browser('webdriver.chrome')
    # browser = Browser('firefox')
    browser.visit(url)
    #time.sleep(5)
    browser.cookies.add({
        'whatever': 'and ever'
    })  #https://splinter.readthedocs.io/en/latest/elements-in-the-page.html
    cookie = browser.cookies.all()
    print browser.is_element_present_by_xpath('//h1')
    print cookie
    #browser.cookies.delete('whatever', 'wherever')  # deletes two cookies
    #browser.cookies.delete()  # deletes all cookies
    #browser.execute_script("$('body').empty()")
    print browser.evaluate_script("4+4") == 8
    #fill in account and password
    if browser.find_by_id('login_frame'):
        with browser.get_iframe('login_frame') as frame:
            frame.find_by_id('switcher_plogin').click()
            print u'输入账号...'
            frame.find_by_id('u').fill(q)
            print u'输入密码...'
            frame.find_by_id('p').fill(p)
            print u'尝试登录...'
            frame.find_by_id('login_button').click()
            print u'完成登录动作...'
Пример #2
0
 def __enter__(self):
     browser = Browser()
     url = BASE_URL + 'login.jsp'
     browser.visit(url)
     with browser.get_iframe('base'):
         browser.fill('loginfoodcoopid', FOOD_COOP_ID)
         browser.fill('loginnickname', LOGIN_NAME)
         browser.fill('loginpassword', PASSWORD)
         browser.find_by_name('').first.click()
         return browser
 def __enter__(self):
     browser = Browser()
     url = BASE_URL + 'login.jsp'
     browser.visit(url)
     with browser.get_iframe('base'):
         browser.fill('loginfoodcoopid', FOOD_COOP_ID)
         browser.fill('loginnickname', LOGIN_NAME)
         browser.fill('loginpassword', PASSWORD)
         browser.find_by_name('').first.click()
         return browser
Пример #4
0
    def __init__(self, jira_url, email, jira_password):
        # type: (str, str, str) -> None
        """Obtain Tempo credentials from JIRA."""
        chrome_options = Options()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        browser = Browser('chrome', options=chrome_options)
        browser.visit(jira_url)
        browser.find_by_id('username').fill(email)
        browser.find_by_id('login-submit').click()
        wait(lambda: browser.find_by_id('password').visible)
        browser.find_by_id('password').fill(jira_password)
        browser.find_by_id('login-submit').click()
        wait(lambda: 'Dashboard.jspa' in browser.url)
        browser.visit(
            jira_url +
            '/plugins/servlet/ac/is.origo.jira.tempo-plugin/tempo-my-work#!')

        def get_tempo_frame():
            # type: () -> Any
            for frame in browser.find_by_tag('iframe'):
                if frame['id'].startswith('is.origo.jira.tempo-plugin'):
                    return frame
            return None

        frame = wait(get_tempo_frame)
        with browser.get_iframe(frame['id']) as iframe:
            tempo_state = json.loads(
                iframe.find_by_id('tempo-container')['data-initial-state'])
        self.session = requests.Session()
        self.session.cookies = add_dict_to_cookiejar(
            self.session.cookies,
            {'tempo_session': tempo_state['tempoSession']})
        self.session.headers.update({
            'Tempo-Context':
            tempo_state['tempoContext'],
            'Tempo-Session':
            tempo_state['tempoSession'],
            'Origin':
            'https://app.tempo.io',
            'Content-Type':
            'application/json',
            'X-Requested-With':
            'XMLHttpRequest',
            'User-Agent':
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
        })
Пример #5
0
def work(info):
    browser = Browser('chrome')
    browser.visit(URL_UCAMPUS)
    btn_login = browser.find_by_text("登录")
    btn_login.click()

    input_username = browser.find_by_name('username')
    input_password = browser.find_by_name('password')
    input_username[0].fill(info[0])
    input_password[0].fill(info[1])
    btn_login = browser.find_by_id("login")
    btn_login.click()

    # 由于原网页的 iframe 延迟渲染,需要等待一下才能使用 get_iframe 方法:
    time.sleep(1)
    try:
        with browser.get_iframe('layui-layer-iframe1') as iframe:
            # 教师账号需要点击span切换tab, 学生账号不需要
            # tab = iframe.find_by_text('我是老师')
            # tab.click()

            # input_num = iframe.find_by_id('teachenum')
            # input_name = iframe.find_by_id('teachename')
            input_num = iframe.find_by_id('studentnum')
            input_name = iframe.find_by_id('studentname')

            input_num[0].fill(info[2])
            input_name[0].fill(info[3])
            # btn_commit = iframe.find_by_id('teachBtn')
            btn_commit = iframe.find_by_id('studenBtn')

            btn_commit.click()  # !!!!!!!!!!!!!!!!!!!!!!!!!
            # btn_commit.right_click()

        msg = 'completed: %s' % str(info)
        print msg
        logger.info(msg)
        time.sleep(1)
        browser.quit()
    except Exception:
        msg = 'error: %s' % str(info)
        print msg
        logger.error(msg)
        time.sleep(1)
        browser.quit()
Пример #6
0
def solve_captcha(url):
    ua = UserAgent()
    browser = Browser("firefox", user_agent=ua.random)
    browser.visit(url)

    try:
        iframe_name = browser.find_by_xpath("//iframe").first["name"]
        with browser.get_iframe(iframe_name) as iframe:
            checkbox = iframe.find_by_xpath(
                '//*[@id="recaptcha-anchor"]').first
            if iframe.is_text_present("I'm not a robot"):
                print("Captcha Loaded")
                print("Waiting....")
                rand_sleep(1, 5)
                print("Mouse Over")
                checkbox.mouse_over()
                rand_sleep(0, 1)
                print("Click")
                checkbox.click()

        while not browser.is_text_present("You are verified"):
            sleep(0.5)

        submit = browser.find_by_xpath("/html/body/form/button").first
        submit.click()

        if "InternalCaptcha" in browser.url:
            print("Failed")
        browser.quit()
    except ElementDoesNotExist:
        print("Failed")
        browser.quit()
    except NoSuchWindowException:
        print("Browser Closed")
        exit(1)
    except WebDriverException:
        print("Browser Broke")
        exit(1)
Пример #7
0
#!/usr/bin/env python3
# -*-coding:utf-8-*-
from splinter import Browser
import time

browser = Browser()
url = "http://mail.163.com/"
browser.visit(url)
time.sleep(2)
with browser.get_iframe('x-URS-iframe') as iframe:
    iframe.find_by_name('email').fill('zyrarter')
    iframe.find_by_name('password').fill('zyr519328')
    iframe.find_by_id('dologin').click()
Пример #8
0
from splinter import Browser    //splinter is the module used for manipulating the browser
import json             //json for encoding the data into the file 
data=dict()         // dictionaries save rollnumber and gpa of the student
browser = Browser('firefox')        //firefox for life
browser.visit('http://www.nitt.edu/prm/ShowResult.htm')     //our results page 
roll=110112000  //starting roll number -1
f=open('gpasem4','w')       // opens the file that has to be written into
while(roll<110112100):  
    #browser.type('TextBox1',roll)
    roll=roll+1 
    with browser.get_iframe('main') as iframe:      // the page is being divided into frames . so select the appropriate frame  
        try :
            iframe.fill('TextBox1',roll)
            iframe.find_by_name('Button1').click()
            browser.find_by_xpath('//select[@id="Dt1"]/option[@value="96"]')._element.click()
            
            name=iframe.find_by_id('LblName')
            gpa=iframe.find_by_id('LblGPA')
            
            data.update({roll:gpa.value})       //appends the new data to our dictionary
            print roll, "done"  // success
            pass
        except :
            print roll, "skipped"       // missing rollnos are skipped
            pass
json.dump(data,f)           //store the dictionary in our file in hdd
f.close()       // very imp . close the file after use. 
Пример #9
0
from splinter import Browser

browser = Browser('chrome')
browser.visit('http://rmdx.testucampus.unipus.cn')

btn_login = browser.find_by_text("登录")
btn_login.click()

input_username = browser.find_by_name('username')
input_password = browser.find_by_name('password')
input_username[0].fill('ucptest2115')
input_password[0].fill('pass2115')
btn_login = browser.find_by_id("login")
btn_login.click()

# 由于原网页的 iframe 延迟渲染,需要等待一下才能使用 get_iframe 方法:
import time
time.sleep(1)

with browser.get_iframe('layui-layer-iframe1') as iframe:
    input_num = iframe.find_by_name('num')
    input_name = iframe.find_by_name('name')
    input_num[0].fill('1000004')
    input_name[0].fill(u'赵四')
    btn_commit = iframe.find_by_id('studenBtn')
    print btn_commit
    # btn_commit.click()  # !!!!!!!!!!!!!!!!!!!!!!!!!

time.sleep(1)
browser.quit()
# except HttpResponseError, e:
# print "Oops, I failed with the status code %s and reason %s" % (e.status_code, e.reason)

# test
# Cookies manipulation
browser.cookies.add({'whatever': 'and ever'})  # add a cookie
browser.cookies.all()  # retrieve all cookies
browser.cookies.delete('mwahahahaha')  # deletes the cookie 'mwahahahaha'
browser.cookies.delete('whatever', 'wherever')  # deletes two cookies
browser.cookies.delete()  # deletes all cookies

# Frames, alerts and prompts
# Using iframes,You can use the get_iframe method and the with statement to interact with iframes. You can pass the
# iframe's name, id, or index to get_ifram

with browser.get_iframe('iframemodal') as iframe:
    iframe.do_stuff()

# Chrome support for alerts and prompts is new in Splinter 0.4.Only webdrivers (Firefox and Chrome) has support for
# alerts and prompts.
alert = browser.get_alert()
alert.text
alert.accept()
alert.dismiss()

prompt = browser.get_alert()
prompt.text
prompt.fill_with('text')
prompt.accept()
prompt.dismiss()
# use the with statement to interacte with both alerts and prompts
Пример #11
0
class KithBot:
    def __init__(self, **info):
        self.base_url = "https://kith.com/"
        self.shop_ext = "collections/"
        self.checkout_ext = "checkout/"
        self.info = info

    def init_browser(self):
        self.b = Browser('chrome')

    def find_product(self):
        try:
            r = requests.get("{}{}{}".format(
                self.base_url, self.shop_ext, self.info["category"])).text
            soup = bs4.BeautifulSoup(r, 'lxml')
            temp_tuple = []
            temp_link = []

            for link in soup.find_all("a", class_="product-card__link"):
                temp_tuple.append((link["href"], link.contents[1].text, link.contents[3].text))

            for i in temp_tuple:
                if i[1] == self.info["product"] and i[2] == self.info["color"]:
                    temp_link.append(i[0])

            self.final_link = list(
                set([x for x in temp_link if temp_link.count(x) == 1]))[0]
            print(self.final_link)
        except requests.ConnectionError as e:
            print("Failed to open url")

    def visit_site(self):
        size = '//div[@data-value='+ self.info["size"] + ']'
        self.b.visit("{}{}".format(self.base_url, str(self.final_link)))
        self.b.find_by_xpath(size).click()
        self.b.find_by_name('add').click()
        self.b.find_by_name('checkout').click()

    def shipping_func(self):
        self.b.fill("checkout[email]", self.info["emailfield"])

        self.b.fill("checkout[shipping_address][first_name]", self.info["firstName"])
        self.b.fill("checkout[shipping_address][last_name]", self.info["lastName"])
        self.b.fill("checkout[shipping_address][address1]", self.info["addressfield"])

        self.b.fill("checkout[shipping_address][city]", self.info["city"])
        self.b.fill("checkout[shipping_address][zip]", self.info["zip"])
        self.b.fill("checkout[shipping_address][phone]", self.info["phonefield"])

        self.b.find_by_id('continue_button').click()

        self.b.find_by_id('continue_button').click()

    def checkout_func(self):
        id0 = self.b.find_by_xpath("//iframe[@class='card-fields-iframe']")[0]['id']
        with self.b.get_iframe(id0) as iframe:
            num = textwrap.wrap(self.info["number"], 4)
            iframe.find_by_name("number").type(num[0])
            iframe.find_by_name("number").type(num[1])
            iframe.find_by_name("number").type(num[2])
            iframe.find_by_name("number").type(num[3])

        id1 = self.b.find_by_xpath("//iframe[@class='card-fields-iframe']")[1]['id']
        with self.b.get_iframe(id1):
            self.b.fill("name", self.info["nameField"])

        id2 = self.b.find_by_xpath("//iframe[@class='card-fields-iframe']")[2]['id']
        with self.b.get_iframe(id2) as iframe:
            num = textwrap.wrap(self.info["expiry"], 2)
            iframe.find_by_name("expiry").type(num[0])
            iframe.find_by_name("expiry").type(num[1])

        id3 = self.b.find_by_xpath("//iframe[@class='card-fields-iframe']")[3]['id']
        with self.b.get_iframe(id3):
            self.b.fill("verification_value", self.info["ccv"])

        # self.b.find_by_id('continue_button').click()

    def main(self):
        self.init_browser()
        self.find_product()
        self.visit_site()
        self.shipping_func()
        self.checkout_func()
Пример #12
0
class control_google():
    def init_browser(self, email, passwd):
        self.state = 'good'
        self.passwd = passwd
        self.login = email

        param = {
            'chrome.noWebsiteTestingDefaults': True,
            'chrome.prefs': {
                'profile.default_content_settings': {
                    'images': 2
                },
            }
        }
        from selenium.webdriver.chrome.options import Options
        options = Options()

        #options.add_argument('--allow-running-insecure-content')
        #options.add_argument('--disable-web-security')
        #options.add_argument('--disk-cache-dir=/var/www/cake2.2.4/app/tmp/cache/selenium-chrome-cache')
        #options.add_argument('--no-referrers')
        #options.add_argument('--window-size=1003,719')
        #options.add_argument('--proxy-server=localhost:8118')
        options.add_argument(
            "'chrome.prefs': {'profile.managed_default_content_settings.images': 2}"
        )

        CHROME = {
            "browserName": "chrome",
            "chrome.prefs": {
                "profile.managed_default_content_settings.images": 2
            },
            "chrome.switches": ["disable-images"],
        }

        self.browser = Browser('chrome', user_agent=useragent)
        #self.browser = Browser('chrome', user_agent=useragent, desired_capabilities=CHROME)

        load_page = 'https://accounts.google.com/ServiceLogin?btmpl=mobile_tier2&hl=ru&service=mobile'
        self.browser.visit(load_page)

        self.browser.find_by_id('Email').first.fill(email + '@gmail.com')
        self.browser.find_by_id('Passwd').first.fill(passwd)

        self.browser.find_by_id('signIn').first.click()

    def _google_hook(self):
        if self.browser.is_element_present_by_id('358'):
            self.browser.find_by_id('358').first.click()

        if self.browser.is_element_present_by_id('link_dismiss'):
            try:
                self.browser.find_by_id('link_dismiss').first.click()
            except:
                pass

        if 'getstarted' in self.browser.url:
            self.browser.back()

        if self.browser.is_element_present_by_id('link_dismiss'):
            self.browser.find_by_id('link_dismiss').first.click()

    def open_profile(self):
        print 'Open light version profile'
        load_page = 'https://plus.google.com/app/basic/%s/about' % self.profile_id
        self.browser.visit(load_page)

    def save_profile(self):
        self.browser.find_by_id('177').first.click()

    def register_google_plus(self, firstName, lastName):
        load_page = 'https://plus.google.com/u/0/?gpsrc=ogpy0&tab=XX'
        self.browser.visit(load_page)

        self.browser.fill('firstName', firstName)
        self.browser.fill('lastName', lastName)

        self.browser.find_by_name('buttonPressed').first.click()

        self.browser.find_by_id('357').first.click()

    def get_profile_id(self):
        load_page = 'https://www.google.com/settings/general-light?ref=/settings/account'
        self.browser.visit(load_page)
        if self.browser.is_element_present_by_xpath('//a[@class="CS"]'):
            profile_link = self.browser.find_by_xpath('//a[@class="CS"]').first
            link_path = profile_link['href']

            return link_path.split('/')[3]
        else:
            return False

    def profile_edit(self, vals):
        self.open_profile()

        print 'Click change profile'
        self.browser.find_by_id('59').first.click()

        #Confirm mobile rules
        self._google_hook()

        self.browser.find_by_name('peWork0').first.fill(vals['company'])
        self.browser.find_by_name('peWorkTitle0').first.fill(vals['position'])
        self.browser.find_by_name('peWorkStartYear0').first.fill(
            vals['year_start'])
        self.browser.find_by_name('peWorkEndYear0').first.fill(
            vals['year_stop'])

        self.browser.find_by_name('peSchool0').first.fill(
            vals['university_name'])
        self.browser.find_by_name('peSchoolMajor0').first.fill(
            vals['field_education_name'])
        self.browser.find_by_name('peSchoolStartYear0').first.fill(
            vals['going_to_college_year'])
        self.browser.find_by_name('peSchoolEndYear0').first.fill(
            vals['after_graduation_year'])

        self.browser.find_by_name('pePlaceLived0').first.fill(
            vals['place_lived'])

        self.browser.find_by_name('pePlaceLivedIsCurrent').first.check()

        self.browser.find_by_name('peGender').first.select("1")

        print 'Done profile_edit'

        self.save_profile()

    def change_photo(self, photo_path):
        self.open_profile()

        print 'Click change profile'
        self.browser.find_by_id('59').first.click()

        print 'Click change photo'
        self.browser.find_by_id('375').first.click()

        self.browser.attach_file('photo_upload_file_name', self.photo_path)
        print 'Done profile_edit'

        self.browser.find_by_id('314').first.click()

        self.save_profile()

    def change_pass(self, old_pass, new_pass):
        print 'Open password  change page'
        load_page = 'https://accounts.google.com/b/0/EditPasswd?hl=ru'
        self.browser.visit(load_page)

        self.browser.find_by_id('OldPasswd').first.fill(old_pass)
        self.browser.find_by_id('Passwd').first.fill(new_pass)
        self.browser.find_by_id('PasswdAgain').first.fill(new_pass)

        self.browser.find_by_id('save').first.click()
        print 'Done change pass'

    def open_full_plus(self):
        'Print open full Google+'
        load_page = 'https://plus.google.com/u/0/'
        self.browser.visit(load_page)

    def open_full_profile(self):
        self.open_full_plus()
        self._google_hook()

        print 'Click user icon'
        self.browser.find_by_id('gbi4i').first.click()

        print 'Click show profile'
        #self.browser.find_by_id('gbmplp').first.click()
        self.browser.find_by_xpath(
            '//a[@class="gbqfb gbiba gbp1"]').first.click()

    def change_name(self, firstName, lastName):
        self.open_full_plus()
        self.open_full_profile()

        print 'Click change name'
        time.sleep(5)
        self.browser.find_by_xpath(
            '//div[@guidedhelpid="profile_name"]').first.click()

        print 'Fill values'
        time.sleep(5)
        self.browser.find_by_xpath(
            '//input[@class="l-pR osa g-A-G"]').first.fill(firstName)
        self.browser.find_by_xpath(
            '//input[@class="l-oR Ika g-A-G"]').first.fill(lastName)

        print 'Save results'
        self.browser.find_by_xpath(
            '//*[starts-with(@class,"a-f-e c-b c-b-M nVrMHf nZQKMd h019o")]'
        ).first.click()

        print 'Confirm'
        self.browser.find_by_name('ok').first.click()

    def youtube_hoock(self):
        if 'ServiceLogin?' in self.browser.url:
            print 'ServiceLogin? Hook'
            self.browser.fill('Passwd', self.passwd)
            self.browser.find_by_name('signIn').first.click()
            #self.browser.back()

        if 'create_channel?' in self.browser.url:
            print 'create_channel? Hook'
            self.browser.click_link_by_partial_href('create_channel')
            self.browser.fill('username', self.login)
            self.browser.find_by_id('channel_submit').click()
            self.browser.back()
            self.browser.back()
            self.browser.back()

        if 'select_site?' in self.browser.url:
            print 'select_site? Hook'
            self.browser.find_by_xpath('//input[@type="submit"]').click()
            self.browser.back()
            self.browser.back()

        if 'switch-profile.g?' in self.browser.url:
            print 'switch-profile.g? Hook'
            self.browser.find_by_id('switchButton').click()

    def youtube_like(self, url):
        self.browser.visit(url)
        self.browser.click_link_by_partial_href('action_like=1')

        self.youtube_hoock(url)

        self.browser.find_by_name('action_rate').click()

    def youtube_dislike(self, url):
        self.browser.visit(url)
        self.browser.click_link_by_partial_href('action_dislike=1')

        self.youtube_hoock()

        self.browser.find_by_name('action_rate').click()

    def youtube_comment(self, url, comment):
        self.browser.visit(url)

        self.browser.click_link_by_partial_href('post_comment')

        self.youtube_hoock()
        try:
            self.browser.click_link_by_partial_href('post_comment')
        except:
            pass

        self.youtube_hoock()
        self.browser.fill('comment', comment)
        self.browser.find_by_name('action_comment').click()

        self.youtube_hoock()

    def youtube_subscribe(self, chane_name):
        load_page = 'http://m.youtube.com/user/%s' % chane_name
        self.browser.visit(load_page)

        self.browser.find_by_name('submit')[1].click()

        self.youtube_hoock()

        try:
            self.browser.find_by_name('submit')[1].click()
        except:
            pass

    def google_friend_connector(self):
        #self.browser.click_link_by_partial_href('post_comment')
        pass

    def blogspot_follow(self, url):
        pass

    def get_capture(self):
        cap_element = self.browser.find_by_xpath('//img[@width="300"]').first
        cap_code = recognize_captcha(cap_element['src'])
        self.browser.fill('recaptcha_response_field', cap_code)

    def blogspot_post_plus(self, url):
        self.browser.visit(url)
        frame_name = self.browser.find_by_xpath(
            '//*[starts-with(@name,"I0_")]')[0]['name']
        print frame_name
        with self.browser.get_iframe(frame_name) as iframe:
            #	#self.browser.find_by_xpath('//span[@class="hAa Qo Bg"]').first.click()
            iframe.find_by_xpath('//span[@class="hAa Qo Bg"]').first.click()

    def blogspot_post(self, url, comment):
        self.browser.visit(url)

        with self.browser.get_iframe('comment-editor') as iframe:
            self.browser.fill('commentBody', comment)
            iframe.find_by_id('postCommentSubmit').click()
            self.youtube_hoock()

        with self.browser.get_iframe('comment-editor') as iframe:
            if iframe.is_element_present_by_id('recaptcha_image'):
                self.get_capture()
                iframe.find_by_id('postCommentSubmit').click()

        if 'showComment=' in self.browser.url:
            return True
        else:
            return False

    def google_post_like(self, url):
        self.browser.visit(url)

        if not self.browser.is_element_present_by_name('stupop'):
            self.browser.find_by_id('162').click()
            return True
        else:
            return False

    def google_post_dislike(self, url):
        self.browser.visit(url)

        if self.browser.is_element_present_by_name('stupop'):
            self.browser.find_by_id('162').click()
            return True
        else:
            return False

    def google_post_comment(self, url, comment):
        self.browser.visit(url)

        self.browser.fill('adcp', comment)
        self.browser.find_by_id('110').click()

    def google_post_share(self, url, comment):
        self.browser.visit(url)

        self.browser.find_by_id('396').click()
        self.browser.fill('rpPostMsg', comment)
        self.browser.find_by_id('253').click()

    def google_profile_join(self, id):
        self.browser.visit('https://plus.google.com/app/basic/%s/' % id)

        self.browser.find_by_id('59').click()
        self.circle_join()

    def circle_join(self):
        self.browser.find_by_name('chcccp')[3].click()
        self.browser.find_by_id('49').click()
        self.browser.reload()

    def google_communities_enter(self, id):
        self.browser.visit('https://plus.google.com/u/0/communities/%s/' % id)
        self._google_hook()

    def google_communities_join(self, id):
        self.google_communities_enter(id)

        if self.browser.is_element_present_by_xpath(
                '//*[starts-with(@class,"a-f-e c-b c-b-La")]'):
            self.browser.find_by_xpath(
                '//*[starts-with(@class,"a-f-e c-b c-b-La")]').first.click()

    def google_communities_post(self, id, mess):
        print 'Start  communities post'
        self.google_communities_join(id)
        time.sleep(60)
        #for i in self.browser.find_by_xpath('//a[@class="FW9qdb Wk"]'):
        #`	print i['oid']

        #self.browser.reload()
        self.browser.find_by_xpath(
            '//div[@guidedhelpid="sharebox_textarea"]').first.click()
        self.browser.find_by_xpath('//div[@class="yd editable"]').first.fill(
            mess)
        self.browser.find_by_xpath(
            '//div[@guidedhelpid="sharebutton"]').click()
        time.sleep(60)
        self.browser.find_by_xpath('//div[@class="a-n Ph Hw"]').first.click()
        print '-' * 30
        for i in self.browser.find_by_xpath('//a[@class="FW9qdb Wk"]'):
            print i['oid']

    def google_people_suggested(self):
        self.browser.visit(
            'https://plus.google.com/app/basic/people/suggested?')
        for i in range(10):
            try:
                self.browser.find_by_xpath('//a[@class="vfc"]').first.click()
                self.circle_join()
            except:
                self.browser.visit(
                    'https://plus.google.com/app/basic/people/suggested?')

    def google_grab_comm_members(self, id, qty):
        irr_qty = int((qty - 64) / 20.00) + 3
        print 'Irr qty= %d' % irr_qty

        self.browser.visit(
            'https://plus.google.com/u/0/communities/%s/members' % id)
        ret_arr = []

        js_del_all_img = """
		var images = document.getElementsByTagName('img');
			while(images.length > 0) 
			{
    		images[0].parentNode.removeChild(images[0]);
			}
		"""

        for i in range(irr_qty):
            elem_arr = self.browser.find_by_xpath('//div[@class="ib31if"]')
            print 'Array len %d' % len(elem_arr)
            print i
            print ''
            elem_arr[len(elem_arr) - 2].right_click()
            #self.browser.execute_script(js_del_all_img)
            for elem in elem_arr:
                oid = elem['oid']
                img = self.browser.find_by_xpath('//img[@oid="%s"]' % oid)[0]
                #print img['src']

                if not oid in ret_arr:
                    ret_arr.append(oid)
                    print oid

        f = open('/tmp/google_oid.txt', 'w')
        for s in ret_arr:
            f.write('<item>' + s + '</item>\n')
        f.close()
        print 'Grab done'

    def quit(self):
        self.browser.quit()
Пример #13
0
#!/usr/bin/python
import time
from splinter import Browser
browser = Browser('phantomjs')
# browser = Browser()
browser.visit('file:///home/ethompsy/Projects/spot/html/spotify_test.html')
print browser.url
print browser.html
with browser.get_iframe('player') as iframe:
    iframe.is_element_present_by_id('play-button', wait_time=10)
    print iframe.find_by_id('play-button').first
    iframe.find_by_id('play-button').first.click()
time.sleep(5)
browser.quit()
Пример #14
0
class ChopeBrowser:
    def __init__(self, headless=False):
        self.chrome = Browser('chrome', headless=headless)

    def time_delay(self, time):
        self.chrome.is_element_present_by_name('!@#$%^&*())(*&^%$#@!',
                                               wait_time=time)

    def login(self, usr, pwd, domain='STUDENT'):
        url = 'https://ntupcb.ntu.edu.sg'
        url += '/fbscbs/Account/SignIn?ReturnUrl=%2ffbscbs'
        self.chrome.visit(url)
        dropdown = self.chrome.find_by_tag('option')

        for option in dropdown:
            if option.text == domain:
                option.click()

        self.chrome.fill('Username', usr)
        self.chrome.fill('Password', pwd + '\n')


# PC BOOKING STARTS HERE
# Tries to book the PC of selected type

    def pc_setup(self, usr, pwd, Type):
        self.login(usr, pwd)
        button = self.chrome.find_by_id('tdPcBook')
        button.click()
        time.sleep(2)
        with self.chrome.get_iframe('frmAdminViewControls') as iframe:
            iframe.find_by_id('pnlInsLoc3').click()
        self.type_number(Type)
        data = self.scrape_pc()

        can_book = self.book_pc(data[1], data[2])
        self.chrome.quit()
        return data[0], can_book

    # identify pc type requested
    def type_number(self, Types):
        for i in range(0, 4):
            with self.chrome.get_iframe('frmAdminViewControls') as iframe:
                page = iframe.find_by_id('pnlInsPcGrp' + str(i))
                if page != []:
                    page = page.html
                    page = BeautifulSoup(page, "lxml")
                    page = page.find("span", {
                        "style":
                        "display:inline-block;height:20px;width:80px;"
                    })
                    page = page.get_text()
                    if page == Types:
                        page = iframe.find_by_id('pnlInsPcGrp' +
                                                 str(i)).click()
                        return
        return 0

    # Scrape all PC in the current screen
    def scrape_pc(self):
        with self.chrome.get_iframe('frmSeating') as iframe:
            for i in range(0, 6):

                for j in range(1, 11):
                    btnID = 'grdSeating_tblCol' + str(j) + '_' + str(i)
                    parse = iframe.find_by_id(btnID)
                    if parse == []:
                        return 'no pc', 100, 100
                    if parse != []:
                        color = self.color(parse.html)
                        if (color == '#FFFFFF'):
                            return self.name_pc(parse.html), j, i
        no_pc = 'no pc'
        j = 100
        i = 100
        return no_pc, j, i

    # Identify name of PC
    def name_pc(self, codes):
        soup = BeautifulSoup(codes, "lxml")
        mydivs = soup.findAll("span", {"class": "lblPcName"})
        return mydivs[0].get_text()

    # Check availability of PC, by detecting background color
    def color(self, code):
        soup = BeautifulSoup(code, "lxml")
        tag = soup.findAll('td', {"style": "background-color: #FFFFFF"})
        if tag != []:
            return '#FFFFFF'
        else:
            return 'blabla'

    # Try to book the selected PC
    def book_pc(self, col, row):
        with self.chrome.get_iframe('frmSeating') as iframe:
            if (col != 100) and (row != 100):
                try:
                    time.sleep(1)
                    butt = iframe.find_by_id("grdSeating_divOuterCol" +
                                             str(col) + "_" + str(row))
                    if butt != []:
                        butt.click()
                    time.sleep(1)
                    sub = iframe.find_by_name("btnsumit")
                    sub.click()
                    return "booked"
                except:
                    pyautogui.press('enter')
                    return "cannot book"
        return "cannot book"

    # Initialize booking site until arriving to the booking table
    def first_setup(self):
        button = self.chrome.find_by_id('tdFacilityBook')
        button.click()
        self.chrome.click_link_by_href('#8')
        self.chrome.click_link_by_href('#-1')
        self.chrome.click_link_by_href('/fbscbs/Booking/Create?resourceId=69')
        self.chrome.click_link_by_id('book')
        self.chrome.click_link_by_id('changeResource')
        self.chrome.click_link_by_href('#-1')
        self.chrome.click_link_by_id('book')

    # Eliminates unnecessary booking slots
    def is_registered(event):
        if event.has_class('noShowWhite'):
            return False
        if event.has_class('currentEvent'):
            return False
        return True

    # Adds weekly booked slots for selected facility
    # Each list of weekly bookings contain list of daily bookings
    # each containing lists booked slots, determined by start and end time
    def check_facility(self, evFacilities):
        columnWeek = self.chrome.find_by_css('.wc-event-column')
        evWeek = []
        for columnDay in columnWeek:
            evToday = []
            evList = columnDay.find_by_css('.ui-corner-all')
            for event in evList:
                if not event.has_class('noShowWhite'):
                    if not event.has_class('currentEvent'):
                        event = event.text
                        if not event.find('—') == -1:
                            if event == '':
                                continue
                            evToday.append(event.split('—'))
            evWeek.append(evToday)
        evFacilities.append(evWeek)

    def click_next(self, counter, evFacilities):
        # Recursively check facilities.
        # Choose facility based on counter
        dropdown = self.chrome.find_by_id('ResourceId')
        options = dropdown.find_by_tag('option')
        if counter < len(options):
            nextOption = options[counter]
            nextOption.click()
            self.check_facility(counter, evFacilities)
        else:
            return evFacilities

    # Scrape seats main function
    # OPTIMIZE: by multithreading
    #           and by runnnig multiple browser at once
    def scrape_seats(self, usr, pwd):
        self.login(usr, pwd)
        self.first_setup()
        evFacilities = []
        dropdown = self.chrome.find_by_id('ResourceId')
        options = dropdown.find_by_tag('option')
        optRange = range(len(options))
        for i in optRange:
            opt = options[i]
            nextOption = opt
            nextOption.click()
            self.time_delay(0.2)
            # while loadingTitle.visible:
            #     pass
            evFacilities.append(opt.text)
            self.check_facility(evFacilities)
        self.quit()
        return evFacilities

    def quit(self):
        self.chrome.quit()
Пример #15
0
# which duo authentication device is going to be used
duo_device_num = 0

f = open("discusion.txt", "w", encoding="utf-8")

browser = Browser('chrome', executable_path=chromedriverPath)
# browser = Browser('chrome', headless=False, executable_path=chromedriverPath)
# browser = Browser('firefox',executable_path=geckoPath)
# url = "http://www.google.com"
browser.visit(url)

browser.fill('username', username)
browser.fill('password', password)
browser.find_by_name('submit').click()

with browser.get_iframe('duo_iframe') as iframe:
    device = iframe.find_by_tag('select').first
    options = iframe.find_by_tag('option')
    device.select(options[duo_device_num].value)
    buts = iframe.find_by_tag('button')
    for but in buts:
        if but.visible:
            but.click()
            break

eles = browser.find_by_css(".message.user_content.enhanced")
while eles is None or len(eles) < 3:
    time.sleep(1)
    eles = browser.find_by_css(".message.user_content.enhanced")
    print("waiting duo authentication...")
posts = []
from splinter import Browser
browser=Browser('firefox')
browser.visit('http://www.way2sms.com/content/index.html')
browser.find_by_name('username').fill('mob')
browser.find_by_name('password').fill('pass')
browser.find_by_name('Login').click()
browser.find_by_css('.boo.I4').click()
browser.find_by_id('quicksms').click()

with browser.get_iframe('frame') as iframe:  
    browser.find_by_name('textArea').fill('textttareaarereara')
    browser.find_by_css('.inp').fill('9710101067')
    browser.find_by_name('Send').click()

Пример #17
0
#!/usr/bin/env python3
# -*-coding:utf-8-*-


from splinter import Browser
import time


browser = Browser()
url = 'https://mail.qq.com/'
browser.visit(url)
with browser.get_iframe('login_frame')as iframe:
    iframe.find_by_id('u').fill('976207292')
    iframe.find_by_id('p').fill('z976207292')
    iframe.find_by_id('login_button').click()
Пример #18
0
    if dropbx == True:
        texturedata["dropbox"] = {"project link": dblink}
    texturesubmitdata = bldir + name + "/" + name + "_postdata.json"
    with open(texturesubmitdata, "w") as f:
        json.dump(texturedata, f, sort_keys=False, indent=4, ensure_ascii=False)
        # prepare browser for forum post
        # parse paragraph to it's final form to be injected via javascript.
        # fill subject line of forum.
        # test for autosubmit status.
    bbcodesubmitdata = bldir + name + "/" + name + "_bbctemp.txt"
    with open(bbcodesubmitdata, "w") as f:
        f.write(template.bbtemplate(texturedata))
    if autosubmit == True:
        pblogin()
        browser.fill("subject", nname)
        with browser.get_iframe(2) as iframe:
            iframe.execute_script(
                """

var forumBody = '%s';
document.getElementsByTagName('body')[0].innerHTML = forumBody;

"""
                % (final)
            )
            # test whether or not a new page with the texture name has been submitted before moving on, note that splinter does not automatically press submit at the moment, this will change in the future when the script is close to 100% accurate.
        pagename = nname + " | Melee HD"
        pagetitle = browser.title
        while pagetitle != pagename:
            pagetitle = browser.title
            time.sleep(1)
Пример #19
0
class SplinterBrowserDriver(BaseBrowserDriver):
    """
        This is a BrowserDriver for splinter
        (http://splinter.cobrateam.info)
        that implements the BaseBrowserDriver API.

        To use it, you must have splinter installed on your env.

        For itself it's a browser driver that supports multiple browsing
        technologies such as selenium, phantomjs, zope, etc.
    """

    driver_name = 'splinter'

    def __init__(self):
        super(SplinterBrowserDriver, self).__init__()
        if not splinter_available:
            raise ImportError(
                "In order to use splinter Base Driver you have to install it. "
                "Check the instructions at http://splinter.cobrateam.info")
        self._browser = Browser(config.default_browser)

    def _handle_empty_element_action(self, element):
        if not element:
            raise ActionNotPerformableException(
                "The action couldn't be perfomed because the element couldn't "
                "be found; Try checking if your element"
                "selector is correct and if the page is loaded properly.")

    @property
    def page_url(self):
        return self._browser.url

    @property
    def page_source(self):
        return self._browser.html

    @property
    def page_title(self):
        return self._browser.title

    def open_url(self, url):
        self._browser.driver.get(url)

    def quit(self):
        return self._browser.quit()

    def is_element_visible(self, element):
        return element.visible

    def get_element_text(self, element):
        return element.text

    def get_element_by_xpath(self, selector):
        return self._browser.find_by_xpath(selector)

    def get_element_by_css(self, selector):
        return self._browser.find_by_css(selector)

    def get_element_by_id(self, selector):
        return self._browser.find_by_id(selector)

    def get_element_by_tag(self, selector):
        return self._browser.find_by_tag(selector)

    @element_action
    def type(self, element, text, slowly=False):
        return element.type(text, slowly)

    @element_action
    def fill(self, element, text):
      return element.fill(text)

    @element_action
    def clear(self, element):
      self.fill(element, '')

    @element_action
    def click(self, element):
        return element.click()

    @element_action
    def check(self, element):
        return element.check()

    @element_action
    def uncheck(self, element):
        return element.uncheck()

    @element_action
    def mouse_over(self, element):
        return element.mouse_over()

    @element_action
    def mouse_out(self, element):
        return element.mouse_out()

    def reload(self):
        return self._browser.reload()

    def go_back(self):
        return self._browser.back()

    def go_forward(self):
        return self._browser.forward()

    def execute_script(self, script):
        return self._browser.evaluate_script(script)

    def get_iframe(self, iframe_id):
        return self._browser.get_iframe(iframe_id)

    def get_alert(self):
        return self._browser.get_alert()

    def attach_file(self, input_name, file_path):
        return self._browser.attach_file(input_name, file_path)

    def wait_pageload(self, timeout=30):
        wait_interval = 0.05
        elapsed = 0

        while self.execute_script('document.readyState') != 'complete':
            self.wait(wait_interval)
            elapsed += wait_interval

            if elapsed > timeout:
                raise PageNotLoadedException

    def click_and_wait(self, element, timeout=30):
        self.click(element)
        self.wait_pageload(timeout)
Пример #20
0
class MovieCheckin:

    USERNAME = '******'
    PASSWORD = '******'
    CITY = 'Linköping'
    CITY_ALIAS = 'Filmstaden Linköping'
    SEEN_MOVIES_FILENAME = 'seen_movies.txt'

    def __init__(self):
        self.manager = SFApiManager()
        self.seen_movies = self.get_seen_movies()
        print("{} - Starting MovieCheckinManager..".format(
            datetime.datetime.now()))
        print("List of Ids for seen movies: {}".format(self.seen_movies))
        self.todays_show = None
        self.scheduler = BlockingScheduler(timezone="Europe/Stockholm")
        self.scheduler.add_job(self.reset,
                               'cron',
                               day_of_week='mon-sun',
                               hour=8,
                               minute=0,
                               second=00)
        self.browser = Browser('firefox')
        self.scheduler.start()

    def reset(self):
        print("{} - Job: time to select a new movie".format(
            datetime.datetime.now()))
        self.todays_show = self.select_show()
        show_length = self.todays_show.get_show_length()
        self.todays_show_end = self.todays_show.time + timedelta(
            minutes=show_length + 10)
        print("Selected movie {} which begins at {} and ends at {}".format(
            self.todays_show.movie.title, self.todays_show.time,
            self.todays_show_end))
        five_minutes_before_movie_start = self.todays_show.time - timedelta(
            minutes=5)
        print("Registering job to run @ {}".format(
            five_minutes_before_movie_start))
        self.scheduler.add_job(self.get_taken_seat,
                               'date',
                               run_date=five_minutes_before_movie_start,
                               id='seat_job')

    def get_taken_seat(self):
        print(
            "{} - Job: five minutes before show starts, selecting taken seat.."
            .format(datetime.datetime.now()))
        self.seat = self.todays_show.get_taken_seat()
        if self.seat != None:
            print("Selected taken seat: {}".format(self.seat))
            print("Registering job to run @ {}".format(self.todays_show_end))
            self.scheduler.add_job(self.register,
                                   'date',
                                   run_date=self.todays_show_end,
                                   id='register_job')
        else:
            "No-one purchased tickets to this show, trying another movie.."
            self.reset()

    def get_seen_movies(self):
        seen_movies_file = open(self.SEEN_MOVIES_FILENAME, 'r')
        seen_movies = [id.rstrip() for id in seen_movies_file.readlines()]
        seen_movies_file.close()
        return seen_movies

    def add_seen_movie(self, movie_id):
        self.seen_movies.append(movie_id)
        print("{} - Adding movie with showId {} to file".format(
            datetime.datetime.now(), movie_id))
        try:
            seen_movies_file = open(self.SEEN_MOVIES_FILENAME, 'a')
            seen_movies_file.write("{}\n".format(movie_id))
            seen_movies_file.close()
        except:
            print("Could not add showId to file!")

    def get_todays_lkpg_shows(self):
        date_str = datetime.datetime.today().strftime('%Y-%m-%d')
        lkpg_ncgid = [
            theater.ncg_id for theater in self.manager.get_theaters()
            if theater.city_alias == "LI"
        ][0]
        return self.manager.get_shows(lkpg_ncgid, date_str, date_str)

    def select_show(self):
        todays_unseen_shows = [
            show for show in self.get_todays_lkpg_shows()
            if show.movie.ncg_id not in self.seen_movies
            and show.time > datetime.datetime.now()
        ]
        selected_show = random.choice(todays_unseen_shows)
        selected_show.seats = self.manager.get_seats(
            selected_show.remote_entity_id)
        print("Todays movie selected: {}".format(selected_show.movie.title))
        print("Ncg-Id: {}".format(selected_show.movie.ncg_id))
        print("Id: {}".format(selected_show.remote_entity_id))
        print("Starts: {}".format(selected_show.time))
        return selected_show

    def register(self):
        print("{} - Job: time to register todays show!".format(
            datetime.datetime.now()))
        cityStr = self.CITY
        cinemaStr = self.CITY_ALIAS
        dateStr = self.todays_show.time.strftime("%Y-%m-%d")
        timeStr = self.todays_show.time.strftime("%H:%m")
        roomNr = self.todays_show.remote_entity_id.split("-")[1]
        seatNr = self.seat[1]
        rowNr = self.seat[0]
        movieTitleStr = self.todays_show.movie.title

        url = "http://www3.sf.se/BIOKLUBBEN/LOGGA-IN/"
        count = 0
        while count < 10:  #dunno why one might have to try several times before it works, I blame Geckodriver..
            try:
                self.browser.visit(url)
                with self.browser.get_iframe("Stack") as iframe:
                    print("Loggar in")
                    iframe.fill('ctl00$ContentPlaceHolder1$LoginNameTextBox',
                                self.USERNAME)
                    iframe.fill('ctl00$ContentPlaceHolder1$PasswordTextBox',
                                self.PASSWORD)
                    iframe.click_link_by_text('Logga in')
                with self.browser.get_iframe("Stack") as iframe:
                    print("Fyller i uppgifter")
                    iframe.click_link_by_text("Efterregistrering")
                    iframe.select('ctl00$ContentPlaceHolder1$CinemaNameDDL',
                                  cinemaStr)
                    iframe.fill(
                        'ctl00$ContentPlaceHolder1$TransactionDateTextBox',
                        dateStr)
                    iframe.fill('ctl00$ContentPlaceHolder1$RowNbrTextBox',
                                rowNr)
                    iframe.fill('ctl00$ContentPlaceHolder1$ShowTimeTextBox',
                                timeStr)
                    iframe.select('ctl00$ContentPlaceHolder1$CityNameDDL',
                                  cityStr)
                    iframe.fill('ctl00$ContentPlaceHolder1$SalonIDTextBox',
                                roomNr)
                    iframe.fill('ctl00$ContentPlaceHolder1$MovieNameTextBox',
                                movieTitleStr)
                    iframe.fill('ctl00$ContentPlaceHolder1$ChairNbrTextBox',
                                seatNr)
                    iframe.click_link_by_text("Skicka")
                    print("{} - Sent registration!".format(
                        datetime.datetime.now()))
                    count = 10
                    break
            except:
                count = count + 1
        self.add_seen_movie(self.todays_show.movie.ncg_id)
Пример #21
0
class SplinterBrowserDriver(BaseBrowserDriver):
    """
        This is a BrowserDriver for splinter
        (http://splinter.cobrateam.info)
        that implements the BaseBrowserDriver API.

        To use it, you must have splinter installed on your env.

        For itself it's a browser driver that supports multiple browsing
        technologies such as selenium, phantomjs, zope, etc.
    """

    driver_name = 'splinter'

    def __init__(self):
        super(SplinterBrowserDriver, self).__init__()
        if not splinter_available:
            raise ImportError(
                "In order to use splinter Base Driver you have to install it. "
                "Check the instructions at http://splinter.cobrateam.info")
        self._browser = Browser(config.default_browser)

    def _handle_empty_element_action(self, element):
        if not element:
            raise ActionNotPerformableException(
                "The action couldn't be perfomed because the element couldn't "
                "be found; Try checking if your element"
                "selector is correct and if the page is loaded properly.")

    @property
    def page_url(self):
        return self._browser.url

    @property
    def page_source(self):
        return self._browser.html

    @property
    def page_title(self):
        return self._browser.title

    def open_url(self, url):
        self._browser.driver.get(url)

    def quit(self):
        return self._browser.quit()

    def is_element_visible(self, element):
        return element.visible

    def get_element_text(self, element):
        return element.text

    def get_element_by_xpath(self, selector):
        return self._browser.find_by_xpath(selector)

    def get_element_by_css(self, selector):
        return self._browser.find_by_css(selector)

    def get_element_by_id(self, selector):
        return self._browser.find_by_id(selector)

    def get_element_by_tag(self, selector):
        return self._browser.find_by_tag(selector)

    @element_action
    def type(self, element, text, slowly=False):
        return element.type(text, slowly)

    @element_action
    def fill(self, element, text):
        return element.fill(text)

    @element_action
    def clear(self, element):
        self.fill(element, '')

    @element_action
    def click(self, element):
        return element.click()

    @element_action
    def check(self, element):
        return element.check()

    @element_action
    def uncheck(self, element):
        return element.uncheck()

    @element_action
    def mouse_over(self, element):
        return element.mouse_over()

    @element_action
    def mouse_out(self, element):
        return element.mouse_out()

    def reload(self):
        return self._browser.reload()

    def go_back(self):
        return self._browser.back()

    def go_forward(self):
        return self._browser.forward()

    def execute_script(self, script):
        """This method is deprecated. Use `execute_javascript` instead.
        """
        return self._browser.evaluate_script(script)

    def execute_javascript(self, script):
        return self._browser.evaluate_script(script)

    def get_iframe(self, iframe_id):
        return self._browser.get_iframe(iframe_id)

    def get_alert(self):
        return self._browser.get_alert()

    def attach_file(self, input_name, file_path):
        return self._browser.attach_file(input_name, file_path)

    def wait_pageload(self, timeout=30):
        wait_interval = 0.05
        elapsed = 0

        while self.execute_javascript('document.readyState') != 'complete':
            self.wait(wait_interval)
            elapsed += wait_interval

            if elapsed > timeout:
                raise PageNotLoadedException

    def click_and_wait(self, element, timeout=30):
        self.click(element)
        self.wait_pageload(timeout)
Пример #22
0
class Compass:

    def __init__(self, username='', password='', outdir=''):
        self._username = username
        self._password = password
        self._outdir = outdir

        self._browser = None
        self._record = None


    def quit(self):
        if self._browser:
            self._browser.quit()
            self._browser = None

    def loggin(self):
        prefs = {
            "browser.download.folderList": 2,
            "browser.download.manager.showWhenStarting": False,
            "browser.download.dir": self._outdir,
            "browser.helperApps.neverAsk.saveToDisk": "application/octet-stream,application/msexcel,application/csv"}

        self._browser = Browser('chrome') #, profile_preferences=prefs)

        self._browser.visit('https://compass.scouts.org.uk/login/User/Login')

        self._browser.fill('EM', self._username)
        self._browser.fill('PW', self._password)
        time.sleep(1)
        self._browser.find_by_text('Log in').first.click()

        # Look for the Role selection menu and select my Group Admin role.
        self._browser.is_element_present_by_name(
            'ctl00$UserTitleMenu$cboUCRoles',
            wait_time=30)
        self._browser.select('ctl00$UserTitleMenu$cboUCRoles', '1253644')
        time.sleep(1)

    def wait_then_click_xpath(self, xpath, wait_time=30, frame=None):
        frame = self._browser if frame is None else frame
        while True:
            try:
                if frame.is_element_present_by_xpath(xpath, wait_time=wait_time):
                    frame.find_by_xpath(xpath).click()
                    break
                else:
                    log.warning("Timeout expired waiting for {}".format(xpath))
                    time.sleep(1)
            except:
                log.warning("Caught exception: ", exc_info=True)

    def wait_then_click_text(self, text, wait_time=30, frame=None):
        frame = self._browser if frame is None else frame
        while True:
            if frame.is_text_present(text, wait_time=wait_time):
                frame.click_link_by_text(text)
                break
            else:
                log.warning("Timeout expired waiting for {}".format(text))

    def adult_training(self):
        self.home()

        # Navigate to training page a show all records.
        self.wait_then_click_text('Training')
        time.sleep(1)
        self.wait_then_click_text('Adult Training')
        time.sleep(1)
        self.wait_then_click_xpath('//*[@id="bn_p1_search"]')

    def home(self):
        # Click the logo to take us to the top
        self.wait_then_click_xpath('//*[@alt="Compass Logo"]')
        time.sleep(1)

    def search(self):
        self.home()

        # Click search button
        self.wait_then_click_xpath('//*[@id="mn_SB"]')
        time.sleep(1)

        # Click "Find Member(s)"
        self.wait_then_click_xpath('//*[@id="mn_MS"]')
        time.sleep(1)

        # Navigate to training page a show all records.
        with self._browser.get_iframe('popup_iframe') as i:
            self.wait_then_click_xpath('//*[@id="LBTN2"]', frame=i)
            time.sleep(1)
            self.wait_then_click_xpath('//*[@class="popup_footer_right_div"]/a', frame=i)
            time.sleep(1)

    def lookup_member(self, member_number):
        self.home()

        # Click search button
        self.wait_then_click_xpath('//*[@id="mn_SB"]')
        time.sleep(1)

        xpath = '//*[@id="CNLookup2"]'
        while True:
            try:
                if self._browser.is_element_present_by_xpath(xpath, wait_time=30):
                    self._browser.find_by_xpath(xpath).fill(member_number)
                    break
                else:
                    log.warning("Timeout expired waiting for {}".format(xpath))
                    time.sleep(1)
            except:
                log.warning("Caught exception: ", exc_info=True)

        self.wait_then_click_xpath('//*[@id="mn_QS"]')

    def fetch_table(self, table_id):
        parser = etree.HTMLParser()

        def columns(row):
            return ["".join(_.itertext()) for _ in
                    etree.parse(StringIO(row.html), parser).findall('/*/td')]

        def headers(row):
            return ["".join(_.itertext()) for _ in
                    etree.parse(StringIO(row.html), parser).findall('/*/td')]

        headers_xpath = '//*[@id ="{}"]/thead/*'.format(table_id)
        table_xpath = '//*[@id ="{}"]/tbody/tr[not(@style="display: none;")]'.format(table_id)

        if self._browser.is_element_present_by_xpath(table_xpath, wait_time=5):

            headings = [headers(row) for row
                        in self._browser.find_by_xpath(headers_xpath)][0]

            records = [columns(row) for row
                       in self._browser.find_by_xpath(table_xpath)]

            # Extend the length of each row to the same length as the columns
            records = [row+([None] * (len(headings)-len(row))) for row in records]

            # And add dummy columns if we do not have enough headings
            headings = headings + ["dummy{}".format(_) for _ in range(0,len(records[0]) - len(headings))]

            return pd.DataFrame.from_records(records, columns=headings)

        log.warning("Failed to find table {}".format(table_id))
        return None

    def member_training_record(self, member_number, member_name):
        self.lookup_member(member_number)

        # Select Training record
        self.wait_then_click_xpath('//*[@id="LBTN5"]')

        personal_learning_plans = self.fetch_table('tbl_p5_TrainModules')
        personal_learning_plans['member'] = member_number
        personal_learning_plans['name'] = member_name

        training_record = self.fetch_table('tbl_p5_AllTrainModules')
        training_record['member'] = member_number
        training_record['name'] = member_name

        mandatory_learning = self.fetch_table('tbl_p5_TrainOGL')
        mandatory_learning['member'] = member_number
        mandatory_learning['name'] = member_name

        return personal_learning_plans, personal_learning_plans, mandatory_learning

    def member_permits(self, member_number, member_name):
        self.lookup_member(member_number)

        # Select Permits
        self.wait_then_click_xpath('//*[@id="LBTN4"]')

        permits = self.fetch_table('tbl_p4_permits')
        if permits is not None:
            permits['member'] = member_number
            permits['name'] = member_name

        return permits

    @lru_cache()
    def get_all_adult_trainers(self):

        self.adult_training()

        return self.fetch_table('tbl_p1_results')

    @lru_cache()
    def get_all_group_members(self):

        self.search()

        self._browser.is_element_present_by_xpath('//*[@id = "MemberSearch"]/tbody', wait_time=10)
        time.sleep(1)

        # Hack to ensure that all of the search results loaded.
        for i in range(0, 5):
            self._browser.execute_script(
                'document.getElementById("ctl00_main_working_panel_scrollarea").scrollTop = 100000')
            time.sleep(1)

        return self.fetch_table('MemberSearch')

    def export(self, section):
        # Select the My Scouting link.
        self._browser.is_text_present('My Scouting', wait_time=30)
        self._browser.click_link_by_text('My Scouting')

        # Click the "Group Sections" hotspot.
        self.wait_then_click_xpath('//*[@id="TR_HIER7"]/h2')

        # Clink the link that shows the number of members in the section.
        # This is the one bit that is section specific.
        # We might be able to match on the Section name in the list,
        # which would make it more robust but at present we just hard
        # the location in the list.
        section_map = {
            'garrick': 2,
            'paget': 3,
            'swinfen': 4,
            'brown': 4,
            'maclean': 5,
            'rowallan': 6,
            'somers': 7,
            'boswell': 8,
            'erasmus': 9,
            'johnson': 10
        }
        self.wait_then_click_xpath(
            '//*[@id="TR_HIER7_TBL"]/tbody/tr[{}]/td[4]/a'.format(
                section_map[section.lower()]
            ))

        # Click on the Export button.
        self.wait_then_click_xpath('//*[@id="bnExport"]')

        # Click to say that we want a CSV output.
        self.wait_then_click_xpath(
            '//*[@id="tbl_hdv"]/div/table/tbody/tr[2]/td[2]/input')
        time.sleep(2)

        # Click to say that we want all fields.
        self.wait_then_click_xpath('//*[@id="bnOK"]')

        download_path = os.path.join(self._outdir, 'CompassExport.csv')

        if os.path.exists(download_path):
            log.warn("Removing stale download file.")
            os.remove(download_path)

        # Click the warning.
        self.wait_then_click_xpath('//*[@id="bnAlertOK"]')

        # Browser will now download the csv file into outdir. It will be called
        # CompassExport.

        # Wait for file.
        timeout = 30
        while not os.path.exists(download_path):
            time.sleep(1)
            timeout -= 1
            if timeout <= 0:
                log.warn("Timeout waiting for {} export to download.".fomat(
                    section
                ))
                break

        # rename download file.
        os.rename(download_path,
                  os.path.join(self._outdir, '{}.csv'.format(section)))

        log.info("Completed download for {}.".format(section))

        # Draw breath
        time.sleep(1)

    def load_from_dir(self):
        # Load the records form the set of files in self._outdir.

        log.debug('Loading from {}'.format(self._outdir))

        def get_section(path, section):
            df = pd.read_csv(path, dtype=object, sep=',')
            df['section'] = section
            df['forenames_l'] = [_.lower().strip() for _ in df['forenames']]
            df['surname_l'] = [_.lower().strip() for _ in df['surname']]
            return df

        self._records = pd.DataFrame().append(
            [get_section(os.path.join(self._outdir, section),
                         os.path.splitext(section)[0])
             for section in os.listdir(self._outdir)], ignore_index=True)

    def find_by_name(self, firstname, lastname, section_wanted=None,
                     ignore_second_name=True):
        """Return list of matching records."""

        recs = self._records
        
        if ignore_second_name:
            df = recs[
                (recs.forenames_l.str.lower().str.match(
                        '^{}.*$'.format(firstname.strip(' ')[0].lower().strip()))) &
                  (recs.surname_l == lastname.lower().strip())]
            
        else:
            df = recs[(recs.forenames_l == firstname.lower().strip()) &
                      (recs.surname_l == lastname.lower().strip())]

        if section_wanted is not None:
            df = df[(df['section'] == section_wanted)]

        return df

    def sections(self):
        "Return a list of the sections for which we have data."
        return self._records['section'].unique()

    def all_yp_members_dict(self):
        return {s: members for s, members in self._records.groupby('section')}

    def section_all_members(self, section):
        return [m for i, m in self._records[
            self._records['section'] == section].iterrows()]

    def section_yp_members_without_leaders(self, section):
        return [m for i, m in self._records[
            (self._records['section'] == section) &
            (self._records['role'].isin(
                ['Beaver Scout', 'Cub Scout', 'Scout']))].iterrows()]

    def members_with_multiple_membership_numbers(self):
        return [member for s, member in self._records.groupby(
            ['forenames', 'surname']).filter(
                lambda x: len(x['membership_number'].unique()) > 1).groupby(
                    ['forenames', 'surname', 'membership_number'])]