def download_linke(coords, proxy, port, saveFile, saveMode):

    print proxy, port
    print proxy != ""

    url = "http://www.soda-is.com/eng/services/service_invoke/gui.php?" + "xml_descript=soda_tl.xml&Submit2=Month"

    session = Session()
    session.verify = False

    if proxy != "":
        proxies = {proxy: port}
        session.proxies = proxies

    br = RoboBrowser(session=session, parser="lxml")
    br.open(url)

    linke_form = br.get_forms()[1]

    num = len(coords)
    index = 0

    with open(saveFile, saveMode) as f:
        try:
            for coord in coords:
                inlon, inlat = coord
                linke_form["lat"].value = inlat
                linke_form["lon"].value = inlon

                sf = linke_form.submit_fields.getlist("execute")
                br.submit_form(linke_form, submit=sf[0])

                linke_table = br.find("table", {"cellspacing": "0", "cellpadding": "2"})

                linkes = get_monthly_linke_str(get_linke_values(linke_table))
                s = "%s,%s,%s\n" % (format(inlon, "0.5f"), format(inlat, "0.5f"), linkes)

                if len(s) > 48:
                    f.write(s)
                    print "Done with point %i of %i: (%s, %s)" % (
                        index + 1,
                        num,
                        format(inlon, "0.5f"),
                        format(inlat, "0.5f"),
                    )

                index += 1

                br.back()

            print "DONE!"

        except Exception as e:

            not_dl = list(coords[index:])
            with open(saveFile + "_notdownloaded.txt", "w") as nd:
                for c in not_dl:
                    nd.write("%s,%s\n" % (str(c[0]), str(c[1])))
            print e
 def search_clue(self):
     global count
     for row in self.clue_answer_length_array:
         clue = row[0]
         answer_length = row[1]
         global browser
         browser = RoboBrowser(history=True)
         print('Visiting dictionary.com search page')
         browser.open('http://www.dictionary.com/fun/crosswordsolver')
         form = browser.get_form(action='http://www.dictionary.com/fun/crosswordsolver')
         print('Enter in clue and answer_len')
         form['query'] = clue
         form['l'] = str(answer_length)
         browser.submit_form(form)
         self.store_results()
예제 #3
0
    def print_out(self):

        print('Searching for %s!' % (self.search_var.get()))
        browser = RoboBrowser(history=True)
        browser.open('http://nzbgeek.info')

        form = browser.get_form(action="member.php")
        form['username'].value = self.username_var.get()
        form['password'].value = self.password_var.get()
        browser.submit_form(form)
        pagecount = 1
        searchTerms = self.search_var.get()
        browser.open(
            'https://nzbgeek.info/geekseek.php?moviesgeekseek=1&browsecategory=&browseincludewords='
            + searchTerms + '&p-page=' + str(pagecount) + '#p')

        while (len(browser.select('#browsetable')) > 0):
            titles = [
                tag.string
                for tag in browser.select('.HighlightTVRow2 .title b') if tag
            ]
            links = [
                res.get('href') for res in browser.select('.icon_nzb a') if res
            ]
            for i in range(0, len(titles) - 1):
                if re.search(r'' + self.patternStr_var.get(), titles[i]):
                    self.download_file(titles[i], links[i], self.dirVar)

            pagecount += 1
            browser.open(
                'https://nzbgeek.info/geekseek.php?moviesgeekseek=1&browsecategory=&browseincludewords='
                + searchTerms + '&p-page=' + str(pagecount) + '#p')
        print("Search completed!")
예제 #4
0
def get_det(tid):
    tindex = gid.index(tid)
    rno = rid[tindex]
    pas = pid[tindex]
    print(rno, pas)
    br = RoboBrowser(history=True, parser="html.parser")
    br = RoboBrowser(
        user_agent=
        'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6'
    )
    br.open('http://studentscorner.vardhaman.org')
    form = br.get_form(action="")
    form["rollno"] = rno
    form["wak"] = pas
    br.submit_form(form)
    #br.open("http://studentscorner.vardhaman.org")'''
    print(rno)
    #bot.reply_to(m,"wait")
    br.open("http://studentscorner.vardhaman.org/student_information.php")
    bt = br.parsed()
    th = br.select("th")  #3
    td = br.select("td")  #8
    print("In details " + rno)
    #print(z.geturl())
    #if finalurl != "http://studentscorner.vardhaman.org/":
    try:
        return (str(th[3].text.strip()) + ":" + str(td[8].text.strip()) +
                "\n" + str(th[10].text.strip()) + ":" +
                str(td[17].text.strip()) + "\n" + str(th[29].text.strip()) +
                ":" + (str(td[33].text.strip())) + "\n" +
                str(th[31].text.strip()) + ":" + str(td[35].text.strip())
                )  #details
    except IndexError:
        return ("Something is wrong")
예제 #5
0
def pypi_search(term):
    # TODO: Populate result list with each record as a dictionary
    # searched result.
    result = []
    from robobrowser import RoboBrowser
    br = RoboBrowser(parser="lxml")
    br.open("https://pypi.org/")
    if not br.response.ok:
        raise ValueError("Failed at https://pypi.org/")
    form = br.get_form()
    form["q"] = term
    br.submit_form(form)
    if not br.response.ok:
        raise ValueError("Failed at form submit")

    links = br.select("a.package-snippet")[:5]
    for i, link in enumerate(links):
        rec = {}
        rec["index"] = i
        rec["href"] = link["href"]
        rec["name"] = link.span.text
        rec["description"] = link.p.text
        result.append(rec)

    return result
예제 #6
0
class FSNScraper(object):
    """This module will exists in order to...
	read above. """
    username_field = 'ctl00$ContentPlaceHolder1$tbxUname'
    password_field = 'ctl00$ContentPlaceHolder1$tbxPword'

    def __init__(self):
        super(FSNScraper, self).__init__()
        self.bsr = RoboBrowser()

    def login(self, username, password):
        self.bsr.open('https://myfsn.biz/')
        loginform = self.bsr.get_form()
        loginform[self.username_field].value = username
        loginform[self.password_field].value = password
        self.bsr.submit_form(loginform)

    def get_sr_details(self, srnumber):
        sr_url = 'https://myfsn.biz/SC_Main/SC_SRDetail.aspx?AllowAnyTech=Y&srid=' + srnumber
        self.bsr.open(sr_url)
        this_value = 'ctl00$ContentPlaceHolder1$systemNotesCheckbox'
        event_argument_field = robobrowser.forms.fields.Input(
            '\<input name="__EVENTARGUMENT" value="" \/\>')
        event_target_field = robobrowser.forms.fields.Input(
            '\<input name="__EVENTTARGET" value="ctl00$ContentPlaceHolder1$systemNotesCheckbox" \/\>'
        )
        page_form = self.bsr.get_form()
        page_form.add_field(event_argument_field)
        page_form.add_field(event_target_field)
        self.bsr.submit_form(page_form)
        print(sr_url)
예제 #7
0
def new_token():
    random_string = ''.join(
        random.choice(string.ascii_uppercase + string.digits)
        for _ in range(16))

    from robobrowser import RoboBrowser
    browser = RoboBrowser()
    login_url = 'my_url'
    browser.open('https://github.com/login')
    form = browser.get_form()
    # print(form)
    form["login"].value = "thirstycode"
    form["password"].value = ""
    # print(form)
    browser.submit_form(form)
    browser.open('https://github.com/settings/tokens/new')
    form = browser.get_forms()
    # print(form)
    form[3]["oauth_access[description]"].value = random_string
    form[3]["oauth_access[scopes][]"].value = [
        'repo', 'admin:org', 'admin:public_key', 'admin:repo_hook',
        'admin:org_hook', 'gist', 'notifications', 'user', 'delete_repo',
        'write:discussion', 'admin:gpg_key'
    ]
    browser.submit_form(form[3])

    # print(browser.parsed())
    src = str(browser.parsed())

    start = '<code class="token" id="new-oauth-token">'
    end = '</code>'

    result = re.search('%s(.*)%s' % (start, end), src).group(1)
    return (result)
    print(result)
예제 #8
0
    def submit(self):
        last_id, b, c, d, e = Submit.get_latest_verdict(self.username)

        browser = RoboBrowser(parser='html.parser')
        browser.open('http://codeforces.com/enter')
        enter_form = browser.get_form('enterForm')
        enter_form['handleOrEmail'] = self.username
        enter_form['password'] = self.password
        browser.submit_form(enter_form)

        try:
            checks = list(
                map(lambda x: x.getText()[1:].strip(),
                    browser.select('div.caption.titled')))
            if self.username not in checks:
                click.secho('Login Failed.. Wrong password.', fg='red')
                return
        except Exception as e:
            click.secho('Login Failed.. Maybe wrong id/password.', fg='red')
            return

        browser.open('http://codeforces.com/problemset/submit')
        submit_form = browser.get_form(class_='submit-form')
        submit_form['submittedProblemCode'] = self.prob_id
        submit_form['sourceFile'] = self.inputfile

        browser.submit_form(submit_form)
        if browser.url[-6:] != 'status':
            click.secho(
                'Failed submission, probably you have submit the same file before',
                fg='red')
            return

        Submit.print_verdict(last_id, self.username, 100)
        click.secho('[{0}] submitted ...'.format(self.inputfile), fg='green')
예제 #9
0
class Client(object):
    def __init__(self, username, password):
        self.username = username
        self.password = password
        self.browser = RoboBrowser(history=True, parser='lxml')

    def _open(self, path):
        url = build_path(path)
        self.browser.open(url)
        return self.browser.parsed

    def get_data(self, path, retry=True):
        """Loads the given path making sure that the request is authenticated"""
        response = str(self._open(path))
        if 'Logga ut' not in response:
            if retry:
                self.authenticate(self.username, self.password)
                return self.get_data(path, retry=False)
            raise InvalidCredentials()
        return response

    def authenticate(self, username, password):
        self._open('/')
        login_form = self.browser.get_form()
        login_form['LoginForm_Login'].value = username
        login_form['LoginForm_Password'].value = password
        self.browser.submit_form(login_form)
예제 #10
0
파일: scrapper.py 프로젝트: pks18/acl
def network(user, password, url = 'http://erp.iitbbs.ac.in'):
    browser = RoboBrowser(history=False, parser='html.parser')
    try:
        browser.open(url)
    except:
        NetworkErrorClick()

    try:
        form = browser.get_form(action='login.php')
    except:
        NetworkErrorClick()

    if not form:
        click.echo(click.style('Network error, Unable to fetch form \n', fg='red', bold=True))
        exit(0)

    
    form['email'].value = user
    form['password'].value = password

    try:
        browser.submit_form(form)
    except:
        NetworkErrorClick()

    if (browser.url != 'http://erp.iitbbs.ac.in/home.php'):
        return None

    attendance_link = 'http://erp.iitbbs.ac.in/biometric/list_students.php'

    try:
        browser.open(attendance_link)
        return browser.response.text
    except:
        NetworkErrorClick()
예제 #11
0
    def test_calc_interface(self):
        operation = "5,+,2"
        expected_result = 7

        # Add some result to DB
        requests.post('/'.join((TEST_URL, 'calc')),
            data={'operation':'998,-,888'})

        # Init object
        browser = RoboBrowser(history=True, parser='html.parser')
        browser.open(TEST_URL)

        # Fill calc form
        calc_form = browser.get_form(action='/calc')
        calc_form['operation'] = operation
        browser.submit_form(calc_form)

        # Get result
        result_raw = browser.find(id="result").text
        self.assertEqual(int(result_raw), expected_result)

        # Check result link
        browser.follow_link(browser.find(id='result_link'))
        self.assertEqual((operation, expected_result),
            (browser.find(id="operation").text, int(browser.find(id="result").text)))
예제 #12
0
def get_egfdm_authorization_rb_session(url, data, requests_cache):
    headers = {
        'User-agent':
        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',
        'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3',
        'Accept-Encoding': 'gzip, deflate, br'
    }
    session = requests_cache.CachedSession()
    browser = RoboBrowser()
    browser.session.headers = headers
    browser.session.verify = False
    # encoded_args = urlencode(data)
    browser.open(url)
    form = browser.get_form(action='/act/_login')
    form['logname'].value = data['logname']
    form['password'].value = data['password']
    form['entry_point'].value = data['entry_point']
    form['ctlid2752307'].value = data['ctlid2752307']
    form['login_url'].value = data['login_url']
    # form['lang']
    # url = '{url}/act/_login?login_url={login_url}&entry_point{entry_point}&'.format(url=url, **access.data )
    browser.submit_form(form)

    # response = session.post(url, data=data, headers=headers, verify=False)

    print(browser.response.text)
    print(browser.response.from_cache)

    if browser.response.status_code == 200:
        print(browser.response)
    return browser.session
예제 #13
0
def network(user, password):

    url = 'http://erp.iitbbs.ac.in'
    browser = RoboBrowser(history=False, parser='html.parser')
    try:
        browser.open(url)
    except:
        click.echo(
            click.style(
                'Network error, please check your internet connection and firewall for URL: http://erp.iitbbs.ac.in \n',
                fg='red',
                bold=True))
        exit(0)

    try:
        form = browser.get_form(action='login.php')
    except:
        click.echo(
            click.style(
                'Network error, please check your internet connection and firewall for URL: http://erp.iitbbs.ac.in \n',
                fg='red',
                bold=True))
        exit(0)

    if not form:
        click.echo(
            click.style('Network error, Unable to fetch form \n',
                        fg='red',
                        bold=True))
        exit(0)

    form['email'].value = user
    form['password'].value = password

    try:
        browser.submit_form(form)
    except:
        click.echo(
            click.style(
                'Network error, please check your internet connection and firewall for URL: http://erp.iitbbs.ac.in \n',
                fg='red',
                bold=True))
        exit(0)

    if (browser.url != 'http://erp.iitbbs.ac.in/home.php'):
        return None

    attendance_link = 'http://erp.iitbbs.ac.in/biometric/list_students_date_wise.php'

    try:
        browser.open(attendance_link)
    except:
        click.echo(
            click.style(
                'Network error, please check your internet connection and firewall for URL: http://erp.iitbbs.ac.in \n',
                fg='red',
                bold=True))
        exit(0)

    return browser
예제 #14
0
def login():
	"""Used to login into a server
	returns a robobrowser instance logined in
	"""
	s = requests.Session()
	s.headers['User-Agent'] = 'Mozilla (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.7 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.7'
	browser = RoboBrowser(history=True, parser='html5lib', session=s)

	browser.open('http://en.ogame.gameforge.com')

	loginForm = browser.get_form('loginForm')
	print loginForm['uni'].value
	print loginForm['login'].value
	print loginForm['pass'].value

	loginForm['uni'].value = 'SERVER URL'
	loginForm['login'].value = 'USERNAME'
	loginForm['pass'].value = 'PASSWORD'

	browser.submit_form(loginForm)

	if 'loginError' in browser.url:
		print 'loginError'
		return None

	return browser
def Login():
    username = '******'
    password = '******'

    browser = RoboBrowser(parser='html.parser')
    login_url = 'http://artificialeducation.com/login/'
    browser.open(login_url)
    form = browser.get_form(id='swpm-login-form')
    form['swpm_user_name'].value = username
    form['swpm_password'].value = password
    browser.submit_form(form)
    browser.open('http://artificialeducation.com/login-success-page/')


    soup = BeautifulSoup(str(browser.parsed))

    success_message = soup.findAll('p')
    final_message = ''

    for node in success_message:
        final_message += ''.join(node.findAll(text=True))

    print(final_message)
    if 'Login succeed' in final_message:
        print('Will pass to Main_Storyboard')
    else:
        print('Login/Account Creation failed')
예제 #16
0
def downloadFile(jobId, idOrganism, pathFile, extentionFile, outputValue):
    try:
        resultExistance = confirmExistanceFile(pathFile, idOrganism,
                                               extentionFile)
        if resultExistance == False:
            browser = RoboBrowser()
            login_url = 'http://rast.nmpdr.org/rast.cgi'
            browser.open(login_url)
            form = browser.get_form(id='login_form')
            form['login'].value = 'gresch'
            form['password'].value = 'sequencing'
            browser.submit_form(form)

            browser.open(
                'http://rast.nmpdr.org/rast.cgi?page=JobDetails&job=' +
                str(jobId))
            form = browser.get_form(id='download')
            #try:#Sometimes it didn't work then it did, I don't know why
            form['file'].value = str(idOrganism) + extentionFile
            submit_field = form['do_download']
            submit_field.value = 'Authorize'
            browser.submit_form(form, submit=submit_field)
            f = open(pathFile + str(idOrganism) + extentionFile, "wb")
            f.write(browser.response.content)
            f.close()
            outputValue.put('-1')
    except:
        retults = extentionFile
        outputValue.put(retults)
예제 #17
0
 def pushedbutton(self,b):
     account = self.lineEdit.text()
     pasw = self.lineEdit_3.text()
     #use robobrowser module to manipulate web page 
     browser = RoboBrowser(history = True)
     browser.open('http://web1.cmu.edu.tw/stdinfo/login.asp')
     form1 = browser.get_form(id = 'form1')
     form1['f_id'].value = account
     form1['f_pwd'].value = pasw
     browser.submit_form(form1)
     if browser.state.url == "http://web1.cmu.edu.tw/stdinfo/loginerr.asp":
         self.lineEdit_2.setText('帳號密碼錯了?')
     else:
         link_one = browser.get_link(text = re.compile('.意見調查'))
         browser.follow_link(link_one)
         list = []
         for l in browser.get_links(text = re.compile('.填寫.')): 
             list.append(l)
         list.pop(0)
         for li in list:
             browser.follow_link(li)
             form2 = browser.get_form(id = 'thisform')
             form2['Cos_Q1'].value = '1'
             browser.submit_form(form2)
         self.lineEdit_2.setText('Done!')
def main():

        parser = argparse.ArgumentParser(description='Login to Pixelarity.')
        parser.add_argument("email")
        parser.add_argument("password")
        args = parser.parse_args()

        browser = RoboBrowser()
        templates = get_templates()

        browser.open('https://pixelarity.com/login')

        login_form = browser.get_form(id='ajaxForm1')

        login_form['email'].value = args.email
        login_form['password'].value = args.password

        browser.submit_form(login_form)

        for i in range(len(templates)):
                print('Downloading https://pixelarity.com/' + templates[i] + '/download/html ...')
                request = browser.session.get('https://pixelarity.com/' + templates[i] + '/download/html', stream=True)
                with open('px-' + templates[i] + '.zip', "wb") as temp_zip:
                        temp_zip.write(request.content)
                time.sleep(2)
    def take_action(self, parsed_args):
        config_dir = '~/.kddcup2015-cli'
        config_dir = os.path.expanduser(config_dir)

        if os.path.isdir(config_dir):
            config = ConfigParser.ConfigParser(allow_no_value=True)
            config.readfp(open(config_dir + '/config'))

            if parsed_args.username:
                username = parsed_args.username
            else:
                username = config.get('user', 'username')

            if parsed_args.password:
                password = parsed_args.password
            else:
                password = config.get('user', 'password')

        entry = parsed_args.entry
        message = parsed_args.message

        base = 'https://www.kddcup2015.com'
        login_url = '/'.join([base, 'user-ajaxlogin.html'])
        submit_url = '/'.join([base, 'submission-make.html'])
        submission_url = '/'.join(([base, 'submission.html']))

        browser = RoboBrowser()

        response = browser.session.post(
            login_url, dict(email=username, pwd=password)).json()

        if response['rs'] == 'error':
            self.app.stdout.write(response['msg'])

        browser.open(submit_url)

        form = browser.get_form()

        form['_f'].value = open(entry)

        if message:
            form['description'] = message

        browser.submit_form(form)

        sleep(5)

        browser.open(submission_url)

        html_str = str(browser.parsed)
        html = pq(html_str)

        times = list(map(
            lambda x: datetime_parser.parse(x.text),
            html('.td_result +td+td+td+td')))

        newest_index = times.index(max(times))

        score = html('.td_result')[newest_index * 2].text.strip()
        self.app.stdout.write(score + '\n')
예제 #20
0
def login(handle):
    password = getpass('[Secured] Password of {}: '.format(handle))

    print('> Signing in...')

    try:
        browser = RoboBrowser(parser='lxml')
        browser.open('http://codeforces.com/enter')
        enter_form = browser.get_form('enterForm')
        enter_form['handleOrEmail'] = handle
        enter_form['password'] = password
        browser.submit_form(enter_form)

        checks = list(map(lambda x: x.getText()[
                      1:].strip(), browser.select('div.caption.titled')))

        if handle.lower() not in str(checks).lower():
            print('> !!! Login Failed. Please enter valid credentials')
            return None
        else:
            print('> Success!')
            return browser
    except Exception as e:
        print('>', e)
        return None
예제 #21
0
    def show(self):
        browser = RoboBrowser(parser = 'html.parser')
        browser.open('http://codeforces.com/enter')
        enter_form = browser.get_form('enterForm')
        enter_form['handleOrEmail'] = self.username
        enter_form['password'] = self.password
        browser.submit_form(enter_form)

        try:
            checks = list(map(lambda x: x.getText()[1:].strip(),
                browser.select('div.caption.titled')))
            if self.username not in checks:
                Colour.print('Login Failed.. Wrong password.', Colour.RED)
                return
        except Exception as e:
            Colour.print('Login Failed.. Maybe wrong id/password.', Colour.RED)
            return

        browser.open('http://codeforces.com/friends')
        soup = browser.parsed()[0] # no need of soup
        ftable = soup.findAll('div',{'class':'datatable'})[0].findAll('table')[0].findAll('tr')[1:]

        friends = [x.findAll('td')[1].getText().strip() for x in ftable]
        for f in friends:
            print(f)
예제 #22
0
def get_soup(arrive, depart, hotel, govt):
    if govt is True:
        rateCode = 'GOV'
    else:
        rateCode = 'none'

    browser = RoboBrowser(parser='html.parser')

    browser.open(
        'http://www.marriott.com/reservation/availabilitySearch.mi?propertyCode='
        + hotel['property_code'])

    time.sleep(randint(2, 3))

    form = browser.get_form(
        action='/reservation/availabilitySearch.mi?isSearch=false')

    form['fromDate'].value = arrive
    form['toDate'].value = depart
    form['flexibleDateSearch'] = 'true'
    form['clusterCode'] = rateCode

    # submit form
    browser.submit_form(form)

    time.sleep(randint(2, 3))

    return browser
예제 #23
0
def main(url):
    # morons are too lazy to type out the URL scheme
    if urlparse(url).scheme == '':  # rfc3987: parse(url, rule='URI')['scheme']
        url = 'http://{}'.format(url)

    # And i wonder if it is a valid URL
    try:
        urlparse(url).netloc.split('.')[1]
    except IndexError:
        print(os.environ['NICKNAME'] +
              ', give me a valid URL to shorten. Louge off you skeleton pile..')
        return

    # lay on the force bro.
    browser = RoboBrowser(history=True)
    browser.open('http://ezl.ink/index.php')
    form = browser.get_form(0)
    assert isinstance(form, Form)

    form["url"] = url

    browser.submit_form(form)
    html = browser.parsed

    shorturl = re.findall('http[s]?://ezl.ink/[a-zA-Z0-9]+', str(html))
    print(os.environ['NICKNAME'] + ', shorturl: ' + shorturl[0])
def login_to_cf(username, password):
    """login_to_cf creates a codeforces logged in session using RoboBrowser"""

    logger(
        'info', 'Trying to login into codeforces for the handle : {0}'.format(
            username))

    try:
        browser = RoboBrowser(parser='html.parser')
        browser.open('http://codeforces.com/enter')

        enter_form = browser.get_form('enterForm')
        enter_form['handleOrEmail'] = username
        enter_form['password'] = password
        browser.submit_form(enter_form)

        checks = str(browser.select('div.caption.titled')).count(username)
        if checks == 0 or username == "":
            logger('error', 'Login Failed.. Wrong password.')
            return (False, browser)
    except Exception as e:
        logger('error', 'Login Failed.. Maybe wrong id/password.')
        return (False, browser)

    global sessionUser
    sessionUser = username
    logger('success', 'Login successful, Welcome {0}!'.format(sessionUser))
    return (True, browser)
예제 #25
0
def scrape_cosmo_exam(url,email,password):
    browser = RoboBrowser()
    browser.open(tlink)
    search = browser.get_form()
    search[ 'user[email]' ] = email
    search[ 'user[password]' ] = password
    browser.submit_form(search,submit=search.submit_fields['commit'])

    # main page
    browser.follow_link(browser.find_all('a')[2])
    browser

    #browser.get_links()
    all_links = browser.find_all('a')
    announcements_key = list(filter( lambda x: 'Announcements' in x, all_links ))[0]
    announcement_ind = all_links.index(announcements_key)
    browser.follow_link(browser.find_all('a')[announcement_ind])
    
    #obtaining title objects - tags
    titles =mapper( lambda x:date_extract(1) , browser.find_all('h2'))[0]

    # helper function 2
    def date_extract(ind):
        return list(mapper( lambda x:list(x.children)[1], browser.find_all('h2') ))

    # helper function 3
    def matcher(lst,*matches):
        if not matches:
            matches = ['exam','reminder']
        else:
            matches=matches[0]

        return filterer(lambda x:any(string.lower() in str(x).lower() for string in matches) ,lst)

    return titles
예제 #26
0
def _command_bca(username, password):
    from robobrowser import RoboBrowser

    browser = RoboBrowser(parser='html.parser', user_agent=USER_AGENT)
    hostname = 'https://m.klikbca.com'

    try:
        # Login
        browser.open(f'{hostname}/login.jsp')
        form = browser.get_form(method='post')
        form['value(user_id)'].value = username
        form['value(pswd)'].value = password
        browser.submit_form(form)
        if 'accountstmt' not in browser.response.text:
            raise ValueError('Failed to login.')

        # Get balance then statements
        form = browser.get_form(method='post')
        form.action = 'balanceinquiry.do'
        browser.submit_form(form)
        balances = browser.select('table[cellpadding="5"] td[align="right"] b')
        if not balances:
            raise ValueError('Failed to get balance.')
        result = {'BALANCE': balances[0].contents}
        for i in range(4):
            result = {**result, **_get_bca_statements(browser, i)}
        return result
    finally:
        # Logout
        browser.open(f'{hostname}/authentication.do?value(actions)=logout')
예제 #27
0
    def show(self):
        browser = RoboBrowser(parser = 'html.parser')
        browser.open('http://codeforces.com/enter')
        enter_form = browser.get_form('enterForm')
        enter_form['handleOrEmail'] = self.username
        enter_form['password'] = self.password
        browser.submit_form(enter_form)

        try:
            checks = list(map(lambda x: x.getText()[1:].strip(),
                browser.select('div.caption.titled')))
            if self.username not in checks:
                Colour.print('Login Failed.. Wrong password.', Colour.RED)
                return
        except Exception as e:
            Colour.print('Login Failed.. Maybe wrong id/password.', Colour.RED)
            return

        browser.open('http://codeforces.com/contest/'+self.c_name+'/standings/friends/true')
        soup = browser.parsed()[0] # no need of soup
        ftable = soup.findAll('table',{'class':'standings'})[0].findAll('tr')[1:-1]
        tableh = soup.findAll('table',{'class':'standings'})[0].findAll('tr')[0].findAll('th')

        table_data = [[x.getText().strip() for x in tableh]]
        for friend in ftable:
            row = [x.getText().strip() for x in friend.findAll('td')]
            table_data += [row]

        tt = texttable.Texttable()
        tt.add_rows(table_data)
        tt.set_cols_valign(["b"]*len(tableh))
        print(tt.draw())
예제 #28
0
def get_auth_params():
    login = raw_input("Your login: "******"Your password: "******"https://api.vk.com/oauth/token?grant_type=password&v=5.77&scope=status,friends,photos,audio,video,docs,notes,pages,wall,groups,notifications,messages,market&client_id={cli_id}&client_secret={cli_sec}"
        "&username={login}&password={password}".format(
            cli_id=current_client_id,
            cli_sec=current_client_secret,
            login=login,
            password=password))
    browser = RoboBrowser(history=True)
    browser.open(auth_url)
    if 'error' in browser.response.text:
        if '2fa_app' in browser.response.text:
            redirect_url = json.loads(str(
                browser.response.text))["redirect_uri"]
            browser.open(redirect_url)
            twoFA = raw_input("2factor need, code: ")
            form = browser.get_form()
            form['code'] = twoFA
            browser.submit_form(form)
    if 'success=1' in browser.response.url:
        redirected_url = browser.response.url
    else:
        print(browser.parsed)
        sys.exit()
    aup = urlparse.parse_qs(redirected_url)
    save_auth_params(aup['access_token'][0], aup['user_id'][0])
    return aup['access_token'][0], aup['user_id'][0]
	def lookUpNetCTLPan(self, sequ):
		seq='>seq' + '\n'+ sequ
		browser= RoboBrowser(user_agent='Mozilla/5.0', history=True)
		browser.allow_redirects=True
		browser.session.cookies
		Query="http://tools.immuneepitope.org/stools/netchop/netchop.do?app=netchop"
		browser.open(Query)
		net_form= browser.get_form(action="upload-submit.do")
		net_form
		net_form['sequences'].value=seq
		net_form['formtype'].value='netctlpan_select'
		net_form['length'].value='9'
		net_form['species'].value="human"
		net_form['supertype'].value='A2'
		net_form['allele'].value= "HLA-A02:01" #self.amerLength
		print(net_form)
		net_form.serialize()
		net_form
		
		browser.submit_form(net_form, submit="Submit")
		browser
		print(browser)
		
		
		
		
		table_form=browser.get_form(action="tableViewctlpan.do?thePage=1")
		print(table_form)
		
		return
def get_video_url(url):

    br = RoboBrowser(history=True, parser='lxml')
    br.open(url)

    cn = input('请问是否要转换为中文?(y/n)')
    if not cn:
        cn = 'y'
    if cn == 'y':
        # shift to simplified chinese
        lang = br.get_forms()[0]
        lang['session_language'].options = ['cn_CN']
        lang['session_language'].value = 'cn_CN'
        br.submit_form(lang)

    # get video title
    vid_title = br.find('div', {'id': 'viewvideo-title'}).text.strip()
    print('the video you want to download is: {0}'.format(vid_title))
    print('-----------------------------------------------------------')

    # get video id
    vid_id = re.findall(
        r'\d{6}',
        br.find('a', {
            'href': '#featureVideo'
        }).attrs['onclick'])[0]

    # get real video link
    vid_real_url = 'http://192.240.120.34//mp43/{}.mp4'.format(vid_id)
    return vid_real_url, re.sub(
        """[\s+\.\!\/_,$%^*(+\"\']+|[+——!,。|?、~@#¥%……&*():]+""", " ",
        vid_title).strip()
예제 #31
0
def main():
    # Browse to Rap Genius
    browser = RoboBrowser(history=True)
    browser = RoboBrowser(
        parser="html.parser")  # will get a warning if parser not declared
    browser.open('http://rapgenius.com/')

    # Search for Queen
    form = browser.get_form(action='/search')
    form  # <RoboForm q=>
    form['q'].value = 'queen'
    browser.submit_form(form)

    # Look up the first song
    songs = browser.select('.song_name')
    try:
        browser.follow_link(songs[0])
    except IndexError:
        print("Songs Index doesn't exist!")
        return
    lyrics = browser.select('.lyrics')
    try:
        lyrics[0].text  # \n[Intro]\nIs this the real life...
    except IndexError:
        print("Lyrics Index doesn't exist!")

    # Back to results page
    browser.back()

    # Look up my favorite song
    browser.follow_link('death on two legs')

    # Can also search HTML using regex patterns
    lyrics = browser.find(class_=re.compile(r'\blyrics\b'))
    print(lyrics.text)  # \n[Verse 1]\nYou suck my blood like a leech...
예제 #32
0
def start_crawl(pages=2):
    session = Session()
    session.verify = False
    url = 'https://www.sgs.gov.cn/notice/search/ent_except_list'
    b = RoboBrowser(session=session)
    b.open(url)

    basic_info = pd.DataFrame(columns=['name', 'url', 'ID', 'date'])
    detail_info = pd.DataFrame()
    for i in range(pages):  # 改变这个数字控制爬取页数, 网页限制最大50页
        form = b.get_form(id='formInfo')
        if not form:
            continue
        form['condition.pageNo'].value = str(i + 1)  # 修改表单控制页数
        form['condition.keyword'].value = ''
        try:  # dirty fix...
            b.submit_form(form)
            basic_info = basic_info.append(parse_table(b), ignore_index=True)
        except AttributeError:
            pass

    for url in basic_info['url']:
        detail = get_detail(url)
        if isinstance(detail, pd.DataFrame):
            detail_info = detail_info.append(detail, ignore_index=True)

    return basic_info, detail_info
예제 #33
0
파일: ztools_web.py 프로젝트: marmotcai/qas
def zdz_post010(uid,unam,upas,chdr,ctxt,uhost='http://ziwang.com/'):
    brow = RoboBrowser(history=True,cache=True)
    uexit=uhost+'member.php?action=logout'
    brow.open(uexit);zt.wait(1)
    #
    ulog=uhost+'forum.php' #,'58'  #灌水乐园 
    brow.open(ulog);zt.wait(2);#print('ulog,',ulog)   
    xact="member.php?mod=logging&action=login&loginsubmit=yes&infloat=yes&lssubmit=yes"
    xlog=brow.get_form(action=xact);#print('xlog',xlog)
    if xlog==None:return False
    print('@xlog,',unam,upas)
    #
    xlog['username'].value=unam
    xlog['password'].value=upas
    brow.submit_form(xlog)
    #
    #       http://ziwang.com/forum.php?mod=post&action=newthread&fid=67
    upost0='http://ziwang.com/forum.php?mod=post&action=newthread&fid='
    upost=upost0+uid;#print('@xpost, ',uid,upost);
    x=brow.open(upost);zt.wait(1)
    xact_post='forum.php?mod=post&action=newthread&fid='+uid+'&extra=&topicsubmit=yes'
    xpost=brow.get_form(action=xact_post)
    #print('@xpost, ',xpost)
    #
    xpost['subject'].value,xpost['message'].value=chdr,ctxt
    brow.submit_form(xpost)
    #print('@xpost, ',upost);
    #
    #re_brow,upost,chk_post
    return True
예제 #34
0
class login():

    BROWSER = RoboBrowser()
    USERNAME = ''
    PASSWORD = ''
    URL = ''
    TOKEN = ''
    API_ENDPOINT = ''

    def __init__(self, base_url=None, username=None, password=None):
        '''
        
        Login to sparkmeter website
        
        '''

        if base_url is not None and username is not None and password is not None:
            self.login(base_url, username, password)

    def isOnLogin(self):
        '''
        
        Return true if current login session available
        
        '''
        if self.BROWSER._cursor == -1: return False
        return '/login' not in self.BROWSER.url

    def reLogin(self):
        '''
        
        Relogin again to sparkmeter if current session is outdate/timeout
        
        '''
        self.login(self.URL, self.USERNAME, self.PASSWORD)

    def login(self, base_url, username, password):
        '''
        
        Login action to `base_url` with given `username` and `password`
        
        '''
        try:
            self.BROWSER = RoboBrowser(parser='lxml')
            self.USERNAME = username
            self.PASSWORD = password
            self.URL = base_url

            #1
            self.BROWSER.open(base_url)

            #2
            form = self.BROWSER.get_form(action='/login')
            form['email'].value = username
            form['password'].value = password
            self.BROWSER.submit_form(form)
        except Exception as e:
            print(e)

        return self
예제 #35
0
파일: telp.py 프로젝트: puneethkanna/nvtest
def get_atd(rno, pas, tid):
    if tid in gid:
        tindex = gid.index(tid)
        rno = rid[tindex]
        pas = pid[tindex]
        print(rno, pas)
    else:
        return ("First Login")


#Below code can also be written in if block. This is also correct as else returns below code will not execute.
    br = RoboBrowser(history=True, parser="html.parser")
    br = RoboBrowser(
        user_agent=
        'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6'
    )
    br.open('http://studentscorner.vardhaman.org')
    form = br.get_form(action="")
    form["rollno"] = rno
    form["wak"] = pas
    br.submit_form(form)
    br.open("http://studentscorner.vardhaman.org/student_attendance.php")
    bt = br.parsed()
    th = br.select("th")  #3
    td = br.select("td")  #8
    #print str(th[55].text.strip())+":"+str(th[56].text.strip())#attend
    try:
        for i in range(46, 56):
            if (str(th[i].text.strip()) == "Attendance Percentage"):
                if (finalurl != "http://studentscorner.vardhaman.org/"):
                    return str(th[i].text.strip()) + ":" + str(
                        th[i + 1].text.strip())  #attend
    except IndexError:
        return ("Attendance is Freesed")
예제 #36
0
def sign_in(username, password):
    """
    Signs into the DOH website and sets the global session
    to allow other browser instances to access the cookies
    :param username: the username to login with
    :param password: the password to login with
    """
    # If already logged in, don't log in again
    global global_session
    if global_session is not None:
        return True
    # Create Non-JS browser
    browser = RoboBrowser(parser='html.parser')
    # Open login page
    browser.open('https://doh.arcabc.ca/user/login')
    # Get the login form
    form = browser.get_form(id='user-login')
    # Set the username & password
    form['name'].value = username
    form['pass'].value = password
    # Submit the form
    browser.submit_form(form)
    # If successfully signed in
    h1 = browser.find(class_='page__title')
    if h1.text == username:
        # Set the global session
        global_session = browser.session
        return True
    else:
        return False
예제 #37
0
def valid_login(uname, pwd, proxies=False):
    """
	uname: str
	pwd: str
	proxies: dict ({'http': 'http://my.proxy.com/'})
	"""
    if proxies != False:
        session = Session()
        session.proxies = proxies
        browser = RoboBrowser(session=session, parser='lxml')
    else:
        browser = RoboBrowser(history=True,\
        user_agent='Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101  Firefox/40.1'\
        ,parser='lxml')
    login_url = 'https://www.roblox.com/account/signupredir'
    browser.open(login_url)
    form = browser.get_form(action='https://www.roblox.com/newlogin')
    form['username'].value = uname
    form['password'].value = pwd
    browser.submit_form(form)
    source = str(browser.parsed())
    if "Hello, %s!" % uname in source:
        return True
    else:
        return False
예제 #38
0
def main(argv) :
	browser = RoboBrowser()
	login_url = 'http://localhost:8080/SecuredLogin/index.jsp'
	browser.open(login_url)
	form = browser.get_form(id='frm')
	form['uname'].value = str(sys.argv[1])
	form['passwd'].value = str(sys.argv[2])
	browser.submit_form(form)
예제 #39
0
def login(user, password):
    browser = RoboBrowser(parser="lxml")
    browser.open('http://m.mediavida.com/login.php')
    login = browser.get_form(class_='full')
    login['name'].value = user
    login['password'].value = password
    browser.submit_form(login)
    return browser
예제 #40
0
	def open_site(self, chart_name):
		browser = RoboBrowser(history=True, parser='html.parser')
		browser.open(self.url_start + charts[chart_name])

		form = browser.get_form(action='includes/process.php')
		browser.submit_form(form)

		return browser
예제 #41
0
	def open_site(self, chart_name):
		browser = RoboBrowser(history=True, parser='html.parser')
		browser.open(self.url_start)

		form = browser.get_form(action='https://france99.com/includes/process.php?action=update')
		form['u'] = charts[chart_name]
		browser.submit_form(form)

		return browser		
예제 #42
0
def login(user, passwd):
    browser = RoboBrowser(history=True, parser="html.parser")
    browser.open("https://sistemas.ufsc.br/login")

    form = browser.get_form(id="fm1")
    form["username"].value = user
    form["password"].value = passwd
    browser.submit_form(form)

    return browser
예제 #43
0
def main():

    parser = argparse.ArgumentParser(
        description='Submit codeforces in command line')
    parser.add_argument('user', type=str,
                        help='Your codeforces ID')
    parser.add_argument('prob', type=str,
                        help='Codeforces problem ID (Ex: 33C)')
    parser.add_argument('file', type=str,
                        help='path to the source code')
    args = parser.parse_args()

    user_name = args.user
    last_id, _ = get_submission_data(user_name)

    passwd = getpass()

    browser = RoboBrowser()
    browser.open('http://codeforces.com/enter')

    enter_form = browser.get_form('enterForm')
    enter_form['handle'] = user_name
    enter_form['password'] = passwd
    browser.submit_form(enter_form)

    try:
        checks = list(map(lambda x: x.getText()[1:].strip(),
            browser.select('div.caption.titled')))
        if user_name not in checks:
            print("Login Failed.. probably because you've typed"
                  "a wrong password.")
            return
    except Exception as e:
        print("Login Failed.. probably because you've typed"
              "a wrong password.")
        return 

    browser.open('http://codeforces.com/problemset/submit')
    submit_form = browser.get_form(class_='submit-form')
    submit_form['submittedProblemCode'] = args.prob
    submit_form['sourceFile'] = args.file
    browser.submit_form(submit_form)

    if browser.url[-6:] != 'status':
        print('Your submission has failed, probably '
              'because you have submit the same file before.')
        return

    print('Submitted, wait for result...')
    while True:
        id_, verdict = get_submission_data(user_name)
        if id_ != last_id and verdict != 'TESTING':
            print('Verdict = {}'.format(verdict))
            break
        time.sleep(5)
예제 #44
0
파일: bscrape.py 프로젝트: baldegg/bscrape
def scrape(q):

	query = q
	ph = re.compile('(\(\d{3}\)\ \d{3}-\d{4})')
	ad = re.compile('[A-Z]{2}\ (\d{5})')
	site = re.compile('(?<=\?q=).*(?=&sa)')
	result = {
	'name':'!NO DATA!',
	'address':'!NO DATA!',
	'phone':'!NO DATA!',
	'website':'!NO DATA!',
	'blurb':'!NO DATA!'
	}
	#uses mechanize to submit google search
	browser = RoboBrowser(user_agent='Firefox', parser='html.parser')
	browser.open('http://google.com/')
	
	# Search for Porcupine Tree
	form = browser.get_form(action='/search')
	form                # <RoboForm q=>
	form['q'].value = query

	browser.submit_form(form, form.submit_fields['btnG'])




	result['query']=query
	if browser.find("div", {"class" : "_B5d"}):
		result['name'] = browser.find("div", {"class" : "_B5d"}).text.encode('utf-8')
		stuff = browser.find("div", {"class" : "_uXc"})

		address = stuff.find(text=ad)
		if address:
			result['address']=address.encode('utf-8')

		phone = stuff.find(text=ph)
		if phone:
			result['phone']=phone.encode('utf-8')

		blurb = stuff.find("span")
		if blurb:
			result['blurb'] = blurb.text.encode('utf-8')

		website = stuff.find("a", string="Website")
		if website:
			website = website.get('href').encode('utf-8')
			result['website'] = site.search(website).group()


	print result
	delay = random.randint(5,10)
	print "Waiting " + str(delay) + " seconds..."
	time.sleep(delay)
	return result
예제 #45
0
def scrape_site(url, cookie_file=""):
    global s
    s = Session()
    s.headers['User-Agent'] = 'Mozilla/5.0 (X11; Ubuntu; rv:39.0)'
    s.headers['Accept'] = 'text/html'
    s.headers['Connection'] = 'keep-alive'
    if cookie_file:
        s.cookies = LWPCookieJar(cookie_file)
        try:
            s.cookies.load()
            if not s.cookies._cookies:
                # Cookies have expired
                raise Exception
        except (FileNotFoundError, Exception):
            if os.path.exists(cookie_file):
                os.remove(cookie_file)
            browser = RoboBrowser(session=s,
                                  parser='html5lib',
                                  timeout=10)
            if "sankakucomplex.com" in url:
                url_login = "******"
                form_num = 0
                form_user = "******"
                form_password = "******"
                username = website_logins['sankakucomplex_username']
                password = website_logins['sankakucomplex_password']
                browser.open(url_login)
                form = browser.get_form(form_num)
                form[form_user].value = username
                form[form_password].value = password
                browser.submit_form(form)
                s.cookies.save()
            elif "gelbooru.com" in url:
                url_login = "******"
                form_num = 0
                form_user = "******"
                form_password = "******"
                username = website_logins['gelbooru_username']
                password = website_logins['gelbooru_password']
                browser.open(url_login)
                form = browser.get_form(form_num)
                form[form_user].value = username
                form[form_password].value = password
                browser.submit_form(form)
                s.cookies.save()
    browser = RoboBrowser(session=s,
                          parser='html5lib',
                          timeout=10)
    try:
        browser.open(url)
        return browser
    except:
        # TODO: find what exceptions happens here
        printf("[WARNING] TIMEOUT WITH WEBSITE: {0}".format(url))
        return False
예제 #46
0
def login():
    """Login to evilzone and return a browser object.
    """
    browser = RoboBrowser(history=True)
    browser.open('https://www.evilzone.org/login')
    form = browser.get_form(0)
    assert isinstance(form, Form)

    form["user"] = input("EZ Username: "******"passwrd"] = input("Password: ")

    browser.submit_form(form)
    return browser
예제 #47
0
def loginToESPN(leagueID, year):
    link = 'http://games.espn.go.com/flb/leagueoffice?leagueId='+str(leagueID)+'&seasonId='+str(year)
    br = RoboBrowser(history=True)
    br.open(link)
    try:
        form = br.get_form(action="https://r.espn.go.com/espn/memberservices/pc/login")
        form['username'].value = login.username.value
        form['password'].value = login.password.value
        br.submit_form(form)
        print('\nLogging In\n')
    except:
        print('\nNo need to login!\n')

    return br
def getSongLinks(soundcloudURL):
	browser = RoboBrowser(history=False)

	# Go to SoundFlush and ask to rip the specified track.
	browser.open('http://soundflush.com/')
	form = browser.get_form(id='form_download')
	form['track_url'].value = soundcloudURL
	browser.submit_form(form)

	# Grab the download link and filename from the download page.
	downloadLink = browser.select('a#btn_save')[0]
	downloadURL = downloadLink['href']
	downloadName = downloadLink['download']
	return {'url' : downloadURL, 'name' : downloadName}
예제 #49
0
 def create_calc(self,target,aus):
     browser = RoboBrowser(user_agent='a python robot', history=False)
     browser.open('https://game.planetarion.com/bcalc.pl')
     t = filter(lambda a: target.x == a['x'] and target.y == a['y'] and target.z == a['z'],
                aus)
     if len(t) == 1:
         self.add_au(browser,t[0])
     for au in sorted(aus,key=operator.itemgetter('x','y','z')):
         if len(t) != 1 or target.x != au['x'] or target.y != au['y'] or target.z != au['z']:
             self.add_au(browser,au)
     form = browser.get_form()
     form['def_metal_asteroids']=target.size
     form['action']='save'
     browser.submit_form(form)
     return browser.url
예제 #50
0
class infs_brsr:
    """This browser will have functions useful to someone
    browsing the Infusionsoft front end programatically.
    """

    def __init__(self, appname, username, password, *args, **kwargs):
        self.loggedin=False
        self.browser=RoboBrowser(history=True)
        self.appname=appname
        self.username=username
        self.password=password
        self.baseurl = 'https://' + self.appname + '.infusionsoft.com'

    def openbase(self):
        self.browser.open(self.baseurl)

    def login(self):
        self.openbase()
        loginform = self.browser.get_form()
        loginform.fields['username'].value = self.username
        loginform.fields['password'].value = self.password
        self.browser.submit_form(loginform)
        # This next step is probably a bad idea.  It needs
        # some form of control
        self.browser.follow_link(self.browser.get_links()[1])
        self.loggedin=True

    def getapikey(self):
        if not self.loggedin:
            self.login()
        self.browser.open(self.baseurl + 'app/miscSetting/itemWrapper?systemId=nav.admin&settingModuleName=Application&settingTabName=Application')
        pageSoup = BeautifulSoup(self.browser.response.content, 'html.parser')
        self.apikey=pageSoup.findAll(id='Application_Encrypted_Key:_data')[0].text
        return self.apikey

    def importContactCSV(self, pathToCSV='/home/jlmarks/importme.csv'):
        if not self.loggedin:
            self.login()
        importURL = "https://" + self.appname + ".infusionsoft.com/Import/jumpToWizard.jsp?update=false&profileClass=com.infusion.crm.db.importer.profiles.ContactProfile"
        self.browser.open(importURL)
        frms = self.browser.get_forms()
        for eachform in frms:
            if 'id' in eachform.fields.keys():
                self.thisimportid=eachform['id'].value
                correctform = eachform
        correctform.fields.pop('Back')
        correctform.fields['importFile'].value=open(pathToCSV, 'rb')
        self.browser.submit_form(correctform)
예제 #51
0
class Tracker(object):
    def __init__(self):
        self.browser = RoboBrowser(history=True, parser='html.parser')
        self.login()

    def login(self):
        self.browser.open(TRACKER_LOGIN)
        login_form = self.browser.get_form(id='new_member')

        login_form["member[login]"] = settings.TRACKER_LOGIN
        login_form["member[password]"] = settings.TRACKER_PASSWORD
        self.browser.session.headers['Referer'] = TRACKER
        self.browser.submit_form(login_form)

    def create_event(self, event, description):
        try:
            self.browser.open(TRACKER_EVENT)

            event_form = self.browser.get_form(id='new_event')

            event_form["event[title]"] = event["event_name"]
            event_form["event[contact_name]"] = event["contact"]
            event_form["event[contactemail]"] = event["email"]
            event_form[EVENT_DATE + "[description]"] = "Show"
            event_form[EVENT_DATE + "[location_ids][]"] = "70"

            start_date = event["start_date"]

            event_form[EVENT_DATE + "[startdate(1i)]"] = str(start_date.year)
            event_form[EVENT_DATE + "[startdate(2i)]"] = str(start_date.month)
            event_form[EVENT_DATE + "[startdate(3i)]"] = str(start_date.day)
            event_form[EVENT_DATE + "[startdate(5i)]"] = "00"
            event_form[EVENT_DATE + "[enddate(1i)]"] = str(start_date.year)
            event_form[EVENT_DATE + "[enddate(2i)]"] = str(start_date.month)
            event_form[EVENT_DATE + "[enddate(3i)]"] = str(start_date.day)
            event_form[EVENT_DATE + "[enddate(5i)]"] = "05"
            event_form["event[notes]"] = description

            self.browser.submit_form(event_form)

            if "errors prohibited" in str(self.browser.parsed):
                return self.browser.find(id="errorExplanation")
            else:
                return self.browser.url
        except Exception as e:
            return "EXCEPTION! " + str(e)
예제 #52
0
def get_email_by_cin(cin):
    url = 'http://www.mca.gov.in/mcafoportal/viewCompanyMasterData.do'
    browser = RoboBrowser()
    browser.session.headers['User-Agent'] = random.choice(user_agents)
    browser.open(url)
    form = browser.get_forms()[-1]
    form['companyID'].value = cin
    browser.submit_form(form)
    table = browser.find('table', attrs={'class': 'result-forms'})
    if not table:
        return None
    email_header = table.find('td', text='Email Id')
    if not email_header:
        return None
    email_row = email_header.findNext('td')
    email = str.strip(email_row.text)
    return email.lower()
예제 #53
0
def check(user, section):
    user = user.split(':')
    print 'checking [%s] %s' % (section, user[0])

    br = RoboBrowser(timeout=30, parser='html')
    br.open(config.get(section, 'url'))

    form = br.get_form(id=config.get(section, 'form_id'))
    form[config.get(section, 'username')] = user[0]
    form[config.get(section, 'password')] = user[1]

    br.submit_form(form)

    if config.get(section, 'success_sign') in str(br.select):
        print '!! success', user[0], user[1], section, '!!'
        output.write('%s %s %s\n' % (section, user[0], user[1]))

    print 'checking [%s] %s completed' % (section, user[0])
예제 #54
0
  def submit(self, entry, message=None):
    browser = RoboBrowser(history=True)
    self.read_account()

    # [login]
    browser.open(self.login_url)
    login_form = browser.get_form(action='/account/login')
    login_form['UserName'].value = self.username
    login_form['Password'].value = self.password
    browser.submit_form(login_form)
    myname = browser.select('#header-account')[0].text
    print("[SUBMIT] login as \"%s\" @ %s" % (myname, datetime.now()))

    # [submit]
    browser.open(self.submit_url)
    submit_form = browser.get_form(action='/competitions/submissions/accept')
    submit_form['SubmissionUpload'].value = open(entry, 'r')
    if message: submit_form['SubmissionDescription'] = str(message)
    browser.submit_form(submit_form)
    print("[SUBMIT] submitted @ %s" % datetime.now())
예제 #55
0
def gatherData(user, password):
    baseURL = 'https://sigarra.up.pt/feup/pt/'
    browser = RoboBrowser(history=True, parser='html.parser')
    browser.open(baseURL + 'web_page.Inicial')

    # Gets the login form
    form = browser.get_form(action=re.compile(r'validacao'))

    # Updates the login form with the user credentials
    form['p_user'].value = 'up' + user
    form['p_pass'].value = password

    browser.submit_form(form)

    # Goes to the user profile
    browser.open(baseURL + 'fest_geral.cursos_list?pv_num_unico=' + user)

    # Opens the extended view
    extended = browser.find(title='Visualizar informações no contexto do curso')
    browser.follow_link(extended)

    credits = []
    grades = []

    # For each html class containing grades ("i", "p" and "o"), gather data
    for i in browser.find_all(class_='i'):
        if i.find(class_='n aprovado'):
            credits.append(i.find(class_='k n').text)
            grades.append(i.find(class_='n aprovado').text)

    for j in browser.find_all(class_='p'):
        if j.find(class_='n aprovado'):
            credits.append(j.find(class_='k n').text)
            grades.append(j.find(class_='n aprovado').text)

    for k in browser.find_all(class_='o'):
        if k.find(class_='n aprovado'):
            credits.append(k.find(class_='k n').text)
            grades.append(k.find(class_='n aprovado').text)

    return credits, grades
예제 #56
0
def login_fast():
	from robobrowser import RoboBrowser
	from bs4 import BeautifulSoup as bs

	browser = RoboBrowser(history=True)
	browser.open('https://m.facebook.com/')
	#FORM FILL
	form = browser.get_form()
	form['email'].value=mail
	form['pass'].value=password
	browser.submit_form(form)
	#PARSER (can also be implemented from robobrowser)
	soup = browser.parsed
	try:
		soup = soup.find("label", {"for" : "u_0_0"})
		username = soup.a
		username.clear()
		username = str(username)
		profile = username[username.find('/')+1 : username.find('?')]
		#print (profile)
	except:
		print("wrong credentials!")
예제 #57
0
def scrape_this_page(page_name):
    page = requests.get(page_name)
    if page.status_code == requests.codes.ok:
        # If a folder on the desktop does not already exist for this given artist, create one, then set it as the
        # directory to save images to
        pictures = get_image_urls(
            html.fromstring(page.content).xpath('//a[not(ancestor::div[@class="gr-body"])]/@href'))
        downloaded_images = []
        s = requests.session()
        s.headers.update({'Referer': 'http://www.deviantart.com/'})
        USERAGENTS = (
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1',
            'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:7.0.1) Gecko/20100101',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
            'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)',
            'Opera/9.99 (Windows NT 5.1; U; pl) Presto/9.9.9',
            'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/ Safari/530.5',
            'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/6.0',
            'Mozilla/5.0 (Windows; U; Windows NT 6.1; pl; rv:1.9.1) Gecko/20090624 Firefox/3.5 (.NET CLR 3.5.30729)'
        )
        browser = RoboBrowser(history=False, session=s, tries=3, user_agent=random.choice(USERAGENTS))
        browser.open(
            'https://www.deviantart.com/users/login?ref=http%3A%2F%2Fwww.deviantart.com%2F&remember_me=1')
        form = browser.get_forms()[1]
        form['username'] = '******'
        form['password'] = '******'
        browser.submit_form(form)
        for picture in pictures:
            # Make sure image has not already been downloaded, and that it is not simply a duplicate url with
            # the comments section open
            if picture not in downloaded_images and '#comments' not in picture:
                browser.open(picture)
                deviation_page = str(browser.parsed)
                if download_img(deviation_page, get_folder(page_name)):
                    print("Grabbed " + picture)
                    downloaded_images.append(picture)
    else:
        print('Bad Url')
    return len(downloaded_images)
예제 #58
0
def main(competition, username, password):

    browser = RoboBrowser(history=True, parser="html.parser")
    base = 'https://www.kaggle.com'
    browser.open('/'.join([base, 'account/login']))

    login_form = browser.get_form(action='/account/login')
    login_form['UserName'] = username
    login_form['Password'] = password
    browser.submit_form(login_form)

    browser.open('/'.join([base, 'c', competition, 'data']))
    files = []
    for a_href in browser.get_links():
        if '.zip' in a_href.text:
            files.append(a_href)

    print('...downloading {0} files...'.format(len(files)))
    for f in files:
        request = browser.session.get(base + f.attrs['href'], stream=True)
        with open(f.attrs['name'], "wb") as zip_file:
            zip_file.write(request.content)
예제 #59
-1
def main():
	browser = RoboBrowser(history=True)
	##
	# First to get the stuff off of the general page.  
	#
	# There is the Presented, Handled, abandoned
	browser.open("http://support.infusiontest.com/csdashboard/general.php")
	generalresults = BeautifulSoup(browser.response.content, 'html.parser')
	phone['presented'] = generalresults.find('div', {'id': 'presented'}).find('div', {'class': 'data'}).text.strip('\r\n ')
	phone['abandoned'] = generalresults.find('div', {'id': 'queued'}).find('div', {'class': 'data'}).text.strip('\r\n ')
	phone['handled'] = generalresults.findAll('div', {'id': 'handled'})[0].find('div', {'class': 'data'}).text.strip('\r\n ')
	phone['abandonedpct'] = generalresults.findAll('div', {'id': 'handled'})[1].find('div', {'class': 'data'}).text.strip('\r\n ')
	chat['presented'] = generalresults.find('div', {'id': 'diverted'}).find('div', {'class': 'data'}).text.strip('\r\n ')
	chat['abandoned'] = generalresults.find('div', {'id': 'sla'}).find('div', {'class': 'data'}).text.strip('\r\n ')
	chat['handled'] = generalresults.find('div', {'id': 'abandoned'}).find('div', {'class': 'data'}).text.strip('\r\n ')
	chat['abandonedpct'] = generalresults.find('div', {'id': 'asa'}).find('div', {'class': 'data'}).text.strip('\r\n ')
	browser.open("http://support.infusiontest.com/csdashboard/stats.php")
	statsresults = BeautifulSoup(browser.response.content, 'html.parser')
	phone['asa'] = statsresults.find('div', {'id': 'phone'}).find('div', {'class': 'data'}).text.strip('\r\n ')
	chat['asa'] = statsresults.find('div', {'id': 'phone'}).find('div', {'class': 'data'}).text.strip('\r\n ')
	browser.open("https://docs.google.com/forms/d/1UvD_au-S6YaDGQ-u23Lth5l-JFrrpUiQT6yVFrj64BA/viewform")
	submitform = browser.get_form()	
	submitform.fields['entry.1210668230'].value = '5pm'
	submitform.fields['entry.339838906'].value = phone['asa']
	submitform.fields['entry.335804195'].value = phone['presented']
	submitform.fields['entry.950389349'].value = phone['handled']
	submitform.fields['entry.125377286'].value = phone['abandoned']
	submitform.fields['entry.73700777'].value = phone['abandonedpct']
	submitform.fields['entry.941849183'].value = chat['asa']
	submitform.fields['entry.1083299158'].value = chat['presented']
	submitform.fields['entry.487211652'].value = chat['handled']
	submitform.fields['entry.1724578827'].value = chat['abandoned']
	submitform.fields['entry.1590181783'].value = chat['abandonedpct']
	browser.submit_form(submitform)