def download_linke(coords, proxy, port, saveFile, saveMode): print proxy, port print proxy != "" url = "http://www.soda-is.com/eng/services/service_invoke/gui.php?" + "xml_descript=soda_tl.xml&Submit2=Month" session = Session() session.verify = False if proxy != "": proxies = {proxy: port} session.proxies = proxies br = RoboBrowser(session=session, parser="lxml") br.open(url) linke_form = br.get_forms()[1] num = len(coords) index = 0 with open(saveFile, saveMode) as f: try: for coord in coords: inlon, inlat = coord linke_form["lat"].value = inlat linke_form["lon"].value = inlon sf = linke_form.submit_fields.getlist("execute") br.submit_form(linke_form, submit=sf[0]) linke_table = br.find("table", {"cellspacing": "0", "cellpadding": "2"}) linkes = get_monthly_linke_str(get_linke_values(linke_table)) s = "%s,%s,%s\n" % (format(inlon, "0.5f"), format(inlat, "0.5f"), linkes) if len(s) > 48: f.write(s) print "Done with point %i of %i: (%s, %s)" % ( index + 1, num, format(inlon, "0.5f"), format(inlat, "0.5f"), ) index += 1 br.back() print "DONE!" except Exception as e: not_dl = list(coords[index:]) with open(saveFile + "_notdownloaded.txt", "w") as nd: for c in not_dl: nd.write("%s,%s\n" % (str(c[0]), str(c[1]))) print e
def search_clue(self): global count for row in self.clue_answer_length_array: clue = row[0] answer_length = row[1] global browser browser = RoboBrowser(history=True) print('Visiting dictionary.com search page') browser.open('http://www.dictionary.com/fun/crosswordsolver') form = browser.get_form(action='http://www.dictionary.com/fun/crosswordsolver') print('Enter in clue and answer_len') form['query'] = clue form['l'] = str(answer_length) browser.submit_form(form) self.store_results()
def print_out(self): print('Searching for %s!' % (self.search_var.get())) browser = RoboBrowser(history=True) browser.open('http://nzbgeek.info') form = browser.get_form(action="member.php") form['username'].value = self.username_var.get() form['password'].value = self.password_var.get() browser.submit_form(form) pagecount = 1 searchTerms = self.search_var.get() browser.open( 'https://nzbgeek.info/geekseek.php?moviesgeekseek=1&browsecategory=&browseincludewords=' + searchTerms + '&p-page=' + str(pagecount) + '#p') while (len(browser.select('#browsetable')) > 0): titles = [ tag.string for tag in browser.select('.HighlightTVRow2 .title b') if tag ] links = [ res.get('href') for res in browser.select('.icon_nzb a') if res ] for i in range(0, len(titles) - 1): if re.search(r'' + self.patternStr_var.get(), titles[i]): self.download_file(titles[i], links[i], self.dirVar) pagecount += 1 browser.open( 'https://nzbgeek.info/geekseek.php?moviesgeekseek=1&browsecategory=&browseincludewords=' + searchTerms + '&p-page=' + str(pagecount) + '#p') print("Search completed!")
def get_det(tid): tindex = gid.index(tid) rno = rid[tindex] pas = pid[tindex] print(rno, pas) br = RoboBrowser(history=True, parser="html.parser") br = RoboBrowser( user_agent= 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6' ) br.open('http://studentscorner.vardhaman.org') form = br.get_form(action="") form["rollno"] = rno form["wak"] = pas br.submit_form(form) #br.open("http://studentscorner.vardhaman.org")''' print(rno) #bot.reply_to(m,"wait") br.open("http://studentscorner.vardhaman.org/student_information.php") bt = br.parsed() th = br.select("th") #3 td = br.select("td") #8 print("In details " + rno) #print(z.geturl()) #if finalurl != "http://studentscorner.vardhaman.org/": try: return (str(th[3].text.strip()) + ":" + str(td[8].text.strip()) + "\n" + str(th[10].text.strip()) + ":" + str(td[17].text.strip()) + "\n" + str(th[29].text.strip()) + ":" + (str(td[33].text.strip())) + "\n" + str(th[31].text.strip()) + ":" + str(td[35].text.strip()) ) #details except IndexError: return ("Something is wrong")
def pypi_search(term): # TODO: Populate result list with each record as a dictionary # searched result. result = [] from robobrowser import RoboBrowser br = RoboBrowser(parser="lxml") br.open("https://pypi.org/") if not br.response.ok: raise ValueError("Failed at https://pypi.org/") form = br.get_form() form["q"] = term br.submit_form(form) if not br.response.ok: raise ValueError("Failed at form submit") links = br.select("a.package-snippet")[:5] for i, link in enumerate(links): rec = {} rec["index"] = i rec["href"] = link["href"] rec["name"] = link.span.text rec["description"] = link.p.text result.append(rec) return result
class FSNScraper(object): """This module will exists in order to... read above. """ username_field = 'ctl00$ContentPlaceHolder1$tbxUname' password_field = 'ctl00$ContentPlaceHolder1$tbxPword' def __init__(self): super(FSNScraper, self).__init__() self.bsr = RoboBrowser() def login(self, username, password): self.bsr.open('https://myfsn.biz/') loginform = self.bsr.get_form() loginform[self.username_field].value = username loginform[self.password_field].value = password self.bsr.submit_form(loginform) def get_sr_details(self, srnumber): sr_url = 'https://myfsn.biz/SC_Main/SC_SRDetail.aspx?AllowAnyTech=Y&srid=' + srnumber self.bsr.open(sr_url) this_value = 'ctl00$ContentPlaceHolder1$systemNotesCheckbox' event_argument_field = robobrowser.forms.fields.Input( '\<input name="__EVENTARGUMENT" value="" \/\>') event_target_field = robobrowser.forms.fields.Input( '\<input name="__EVENTTARGET" value="ctl00$ContentPlaceHolder1$systemNotesCheckbox" \/\>' ) page_form = self.bsr.get_form() page_form.add_field(event_argument_field) page_form.add_field(event_target_field) self.bsr.submit_form(page_form) print(sr_url)
def new_token(): random_string = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(16)) from robobrowser import RoboBrowser browser = RoboBrowser() login_url = 'my_url' browser.open('https://github.com/login') form = browser.get_form() # print(form) form["login"].value = "thirstycode" form["password"].value = "" # print(form) browser.submit_form(form) browser.open('https://github.com/settings/tokens/new') form = browser.get_forms() # print(form) form[3]["oauth_access[description]"].value = random_string form[3]["oauth_access[scopes][]"].value = [ 'repo', 'admin:org', 'admin:public_key', 'admin:repo_hook', 'admin:org_hook', 'gist', 'notifications', 'user', 'delete_repo', 'write:discussion', 'admin:gpg_key' ] browser.submit_form(form[3]) # print(browser.parsed()) src = str(browser.parsed()) start = '<code class="token" id="new-oauth-token">' end = '</code>' result = re.search('%s(.*)%s' % (start, end), src).group(1) return (result) print(result)
def submit(self): last_id, b, c, d, e = Submit.get_latest_verdict(self.username) browser = RoboBrowser(parser='html.parser') browser.open('http://codeforces.com/enter') enter_form = browser.get_form('enterForm') enter_form['handleOrEmail'] = self.username enter_form['password'] = self.password browser.submit_form(enter_form) try: checks = list( map(lambda x: x.getText()[1:].strip(), browser.select('div.caption.titled'))) if self.username not in checks: click.secho('Login Failed.. Wrong password.', fg='red') return except Exception as e: click.secho('Login Failed.. Maybe wrong id/password.', fg='red') return browser.open('http://codeforces.com/problemset/submit') submit_form = browser.get_form(class_='submit-form') submit_form['submittedProblemCode'] = self.prob_id submit_form['sourceFile'] = self.inputfile browser.submit_form(submit_form) if browser.url[-6:] != 'status': click.secho( 'Failed submission, probably you have submit the same file before', fg='red') return Submit.print_verdict(last_id, self.username, 100) click.secho('[{0}] submitted ...'.format(self.inputfile), fg='green')
class Client(object): def __init__(self, username, password): self.username = username self.password = password self.browser = RoboBrowser(history=True, parser='lxml') def _open(self, path): url = build_path(path) self.browser.open(url) return self.browser.parsed def get_data(self, path, retry=True): """Loads the given path making sure that the request is authenticated""" response = str(self._open(path)) if 'Logga ut' not in response: if retry: self.authenticate(self.username, self.password) return self.get_data(path, retry=False) raise InvalidCredentials() return response def authenticate(self, username, password): self._open('/') login_form = self.browser.get_form() login_form['LoginForm_Login'].value = username login_form['LoginForm_Password'].value = password self.browser.submit_form(login_form)
def network(user, password, url = 'http://erp.iitbbs.ac.in'): browser = RoboBrowser(history=False, parser='html.parser') try: browser.open(url) except: NetworkErrorClick() try: form = browser.get_form(action='login.php') except: NetworkErrorClick() if not form: click.echo(click.style('Network error, Unable to fetch form \n', fg='red', bold=True)) exit(0) form['email'].value = user form['password'].value = password try: browser.submit_form(form) except: NetworkErrorClick() if (browser.url != 'http://erp.iitbbs.ac.in/home.php'): return None attendance_link = 'http://erp.iitbbs.ac.in/biometric/list_students.php' try: browser.open(attendance_link) return browser.response.text except: NetworkErrorClick()
def test_calc_interface(self): operation = "5,+,2" expected_result = 7 # Add some result to DB requests.post('/'.join((TEST_URL, 'calc')), data={'operation':'998,-,888'}) # Init object browser = RoboBrowser(history=True, parser='html.parser') browser.open(TEST_URL) # Fill calc form calc_form = browser.get_form(action='/calc') calc_form['operation'] = operation browser.submit_form(calc_form) # Get result result_raw = browser.find(id="result").text self.assertEqual(int(result_raw), expected_result) # Check result link browser.follow_link(browser.find(id='result_link')) self.assertEqual((operation, expected_result), (browser.find(id="operation").text, int(browser.find(id="result").text)))
def get_egfdm_authorization_rb_session(url, data, requests_cache): headers = { 'User-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0', 'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3', 'Accept-Encoding': 'gzip, deflate, br' } session = requests_cache.CachedSession() browser = RoboBrowser() browser.session.headers = headers browser.session.verify = False # encoded_args = urlencode(data) browser.open(url) form = browser.get_form(action='/act/_login') form['logname'].value = data['logname'] form['password'].value = data['password'] form['entry_point'].value = data['entry_point'] form['ctlid2752307'].value = data['ctlid2752307'] form['login_url'].value = data['login_url'] # form['lang'] # url = '{url}/act/_login?login_url={login_url}&entry_point{entry_point}&'.format(url=url, **access.data ) browser.submit_form(form) # response = session.post(url, data=data, headers=headers, verify=False) print(browser.response.text) print(browser.response.from_cache) if browser.response.status_code == 200: print(browser.response) return browser.session
def network(user, password): url = 'http://erp.iitbbs.ac.in' browser = RoboBrowser(history=False, parser='html.parser') try: browser.open(url) except: click.echo( click.style( 'Network error, please check your internet connection and firewall for URL: http://erp.iitbbs.ac.in \n', fg='red', bold=True)) exit(0) try: form = browser.get_form(action='login.php') except: click.echo( click.style( 'Network error, please check your internet connection and firewall for URL: http://erp.iitbbs.ac.in \n', fg='red', bold=True)) exit(0) if not form: click.echo( click.style('Network error, Unable to fetch form \n', fg='red', bold=True)) exit(0) form['email'].value = user form['password'].value = password try: browser.submit_form(form) except: click.echo( click.style( 'Network error, please check your internet connection and firewall for URL: http://erp.iitbbs.ac.in \n', fg='red', bold=True)) exit(0) if (browser.url != 'http://erp.iitbbs.ac.in/home.php'): return None attendance_link = 'http://erp.iitbbs.ac.in/biometric/list_students_date_wise.php' try: browser.open(attendance_link) except: click.echo( click.style( 'Network error, please check your internet connection and firewall for URL: http://erp.iitbbs.ac.in \n', fg='red', bold=True)) exit(0) return browser
def login(): """Used to login into a server returns a robobrowser instance logined in """ s = requests.Session() s.headers['User-Agent'] = 'Mozilla (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.7 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.7' browser = RoboBrowser(history=True, parser='html5lib', session=s) browser.open('http://en.ogame.gameforge.com') loginForm = browser.get_form('loginForm') print loginForm['uni'].value print loginForm['login'].value print loginForm['pass'].value loginForm['uni'].value = 'SERVER URL' loginForm['login'].value = 'USERNAME' loginForm['pass'].value = 'PASSWORD' browser.submit_form(loginForm) if 'loginError' in browser.url: print 'loginError' return None return browser
def Login(): username = '******' password = '******' browser = RoboBrowser(parser='html.parser') login_url = 'http://artificialeducation.com/login/' browser.open(login_url) form = browser.get_form(id='swpm-login-form') form['swpm_user_name'].value = username form['swpm_password'].value = password browser.submit_form(form) browser.open('http://artificialeducation.com/login-success-page/') soup = BeautifulSoup(str(browser.parsed)) success_message = soup.findAll('p') final_message = '' for node in success_message: final_message += ''.join(node.findAll(text=True)) print(final_message) if 'Login succeed' in final_message: print('Will pass to Main_Storyboard') else: print('Login/Account Creation failed')
def downloadFile(jobId, idOrganism, pathFile, extentionFile, outputValue): try: resultExistance = confirmExistanceFile(pathFile, idOrganism, extentionFile) if resultExistance == False: browser = RoboBrowser() login_url = 'http://rast.nmpdr.org/rast.cgi' browser.open(login_url) form = browser.get_form(id='login_form') form['login'].value = 'gresch' form['password'].value = 'sequencing' browser.submit_form(form) browser.open( 'http://rast.nmpdr.org/rast.cgi?page=JobDetails&job=' + str(jobId)) form = browser.get_form(id='download') #try:#Sometimes it didn't work then it did, I don't know why form['file'].value = str(idOrganism) + extentionFile submit_field = form['do_download'] submit_field.value = 'Authorize' browser.submit_form(form, submit=submit_field) f = open(pathFile + str(idOrganism) + extentionFile, "wb") f.write(browser.response.content) f.close() outputValue.put('-1') except: retults = extentionFile outputValue.put(retults)
def pushedbutton(self,b): account = self.lineEdit.text() pasw = self.lineEdit_3.text() #use robobrowser module to manipulate web page browser = RoboBrowser(history = True) browser.open('http://web1.cmu.edu.tw/stdinfo/login.asp') form1 = browser.get_form(id = 'form1') form1['f_id'].value = account form1['f_pwd'].value = pasw browser.submit_form(form1) if browser.state.url == "http://web1.cmu.edu.tw/stdinfo/loginerr.asp": self.lineEdit_2.setText('帳號密碼錯了?') else: link_one = browser.get_link(text = re.compile('.意見調查')) browser.follow_link(link_one) list = [] for l in browser.get_links(text = re.compile('.填寫.')): list.append(l) list.pop(0) for li in list: browser.follow_link(li) form2 = browser.get_form(id = 'thisform') form2['Cos_Q1'].value = '1' browser.submit_form(form2) self.lineEdit_2.setText('Done!')
def main(): parser = argparse.ArgumentParser(description='Login to Pixelarity.') parser.add_argument("email") parser.add_argument("password") args = parser.parse_args() browser = RoboBrowser() templates = get_templates() browser.open('https://pixelarity.com/login') login_form = browser.get_form(id='ajaxForm1') login_form['email'].value = args.email login_form['password'].value = args.password browser.submit_form(login_form) for i in range(len(templates)): print('Downloading https://pixelarity.com/' + templates[i] + '/download/html ...') request = browser.session.get('https://pixelarity.com/' + templates[i] + '/download/html', stream=True) with open('px-' + templates[i] + '.zip', "wb") as temp_zip: temp_zip.write(request.content) time.sleep(2)
def take_action(self, parsed_args): config_dir = '~/.kddcup2015-cli' config_dir = os.path.expanduser(config_dir) if os.path.isdir(config_dir): config = ConfigParser.ConfigParser(allow_no_value=True) config.readfp(open(config_dir + '/config')) if parsed_args.username: username = parsed_args.username else: username = config.get('user', 'username') if parsed_args.password: password = parsed_args.password else: password = config.get('user', 'password') entry = parsed_args.entry message = parsed_args.message base = 'https://www.kddcup2015.com' login_url = '/'.join([base, 'user-ajaxlogin.html']) submit_url = '/'.join([base, 'submission-make.html']) submission_url = '/'.join(([base, 'submission.html'])) browser = RoboBrowser() response = browser.session.post( login_url, dict(email=username, pwd=password)).json() if response['rs'] == 'error': self.app.stdout.write(response['msg']) browser.open(submit_url) form = browser.get_form() form['_f'].value = open(entry) if message: form['description'] = message browser.submit_form(form) sleep(5) browser.open(submission_url) html_str = str(browser.parsed) html = pq(html_str) times = list(map( lambda x: datetime_parser.parse(x.text), html('.td_result +td+td+td+td'))) newest_index = times.index(max(times)) score = html('.td_result')[newest_index * 2].text.strip() self.app.stdout.write(score + '\n')
def login(handle): password = getpass('[Secured] Password of {}: '.format(handle)) print('> Signing in...') try: browser = RoboBrowser(parser='lxml') browser.open('http://codeforces.com/enter') enter_form = browser.get_form('enterForm') enter_form['handleOrEmail'] = handle enter_form['password'] = password browser.submit_form(enter_form) checks = list(map(lambda x: x.getText()[ 1:].strip(), browser.select('div.caption.titled'))) if handle.lower() not in str(checks).lower(): print('> !!! Login Failed. Please enter valid credentials') return None else: print('> Success!') return browser except Exception as e: print('>', e) return None
def show(self): browser = RoboBrowser(parser = 'html.parser') browser.open('http://codeforces.com/enter') enter_form = browser.get_form('enterForm') enter_form['handleOrEmail'] = self.username enter_form['password'] = self.password browser.submit_form(enter_form) try: checks = list(map(lambda x: x.getText()[1:].strip(), browser.select('div.caption.titled'))) if self.username not in checks: Colour.print('Login Failed.. Wrong password.', Colour.RED) return except Exception as e: Colour.print('Login Failed.. Maybe wrong id/password.', Colour.RED) return browser.open('http://codeforces.com/friends') soup = browser.parsed()[0] # no need of soup ftable = soup.findAll('div',{'class':'datatable'})[0].findAll('table')[0].findAll('tr')[1:] friends = [x.findAll('td')[1].getText().strip() for x in ftable] for f in friends: print(f)
def get_soup(arrive, depart, hotel, govt): if govt is True: rateCode = 'GOV' else: rateCode = 'none' browser = RoboBrowser(parser='html.parser') browser.open( 'http://www.marriott.com/reservation/availabilitySearch.mi?propertyCode=' + hotel['property_code']) time.sleep(randint(2, 3)) form = browser.get_form( action='/reservation/availabilitySearch.mi?isSearch=false') form['fromDate'].value = arrive form['toDate'].value = depart form['flexibleDateSearch'] = 'true' form['clusterCode'] = rateCode # submit form browser.submit_form(form) time.sleep(randint(2, 3)) return browser
def main(url): # morons are too lazy to type out the URL scheme if urlparse(url).scheme == '': # rfc3987: parse(url, rule='URI')['scheme'] url = 'http://{}'.format(url) # And i wonder if it is a valid URL try: urlparse(url).netloc.split('.')[1] except IndexError: print(os.environ['NICKNAME'] + ', give me a valid URL to shorten. Louge off you skeleton pile..') return # lay on the force bro. browser = RoboBrowser(history=True) browser.open('http://ezl.ink/index.php') form = browser.get_form(0) assert isinstance(form, Form) form["url"] = url browser.submit_form(form) html = browser.parsed shorturl = re.findall('http[s]?://ezl.ink/[a-zA-Z0-9]+', str(html)) print(os.environ['NICKNAME'] + ', shorturl: ' + shorturl[0])
def login_to_cf(username, password): """login_to_cf creates a codeforces logged in session using RoboBrowser""" logger( 'info', 'Trying to login into codeforces for the handle : {0}'.format( username)) try: browser = RoboBrowser(parser='html.parser') browser.open('http://codeforces.com/enter') enter_form = browser.get_form('enterForm') enter_form['handleOrEmail'] = username enter_form['password'] = password browser.submit_form(enter_form) checks = str(browser.select('div.caption.titled')).count(username) if checks == 0 or username == "": logger('error', 'Login Failed.. Wrong password.') return (False, browser) except Exception as e: logger('error', 'Login Failed.. Maybe wrong id/password.') return (False, browser) global sessionUser sessionUser = username logger('success', 'Login successful, Welcome {0}!'.format(sessionUser)) return (True, browser)
def scrape_cosmo_exam(url,email,password): browser = RoboBrowser() browser.open(tlink) search = browser.get_form() search[ 'user[email]' ] = email search[ 'user[password]' ] = password browser.submit_form(search,submit=search.submit_fields['commit']) # main page browser.follow_link(browser.find_all('a')[2]) browser #browser.get_links() all_links = browser.find_all('a') announcements_key = list(filter( lambda x: 'Announcements' in x, all_links ))[0] announcement_ind = all_links.index(announcements_key) browser.follow_link(browser.find_all('a')[announcement_ind]) #obtaining title objects - tags titles =mapper( lambda x:date_extract(1) , browser.find_all('h2'))[0] # helper function 2 def date_extract(ind): return list(mapper( lambda x:list(x.children)[1], browser.find_all('h2') )) # helper function 3 def matcher(lst,*matches): if not matches: matches = ['exam','reminder'] else: matches=matches[0] return filterer(lambda x:any(string.lower() in str(x).lower() for string in matches) ,lst) return titles
def _command_bca(username, password): from robobrowser import RoboBrowser browser = RoboBrowser(parser='html.parser', user_agent=USER_AGENT) hostname = 'https://m.klikbca.com' try: # Login browser.open(f'{hostname}/login.jsp') form = browser.get_form(method='post') form['value(user_id)'].value = username form['value(pswd)'].value = password browser.submit_form(form) if 'accountstmt' not in browser.response.text: raise ValueError('Failed to login.') # Get balance then statements form = browser.get_form(method='post') form.action = 'balanceinquiry.do' browser.submit_form(form) balances = browser.select('table[cellpadding="5"] td[align="right"] b') if not balances: raise ValueError('Failed to get balance.') result = {'BALANCE': balances[0].contents} for i in range(4): result = {**result, **_get_bca_statements(browser, i)} return result finally: # Logout browser.open(f'{hostname}/authentication.do?value(actions)=logout')
def show(self): browser = RoboBrowser(parser = 'html.parser') browser.open('http://codeforces.com/enter') enter_form = browser.get_form('enterForm') enter_form['handleOrEmail'] = self.username enter_form['password'] = self.password browser.submit_form(enter_form) try: checks = list(map(lambda x: x.getText()[1:].strip(), browser.select('div.caption.titled'))) if self.username not in checks: Colour.print('Login Failed.. Wrong password.', Colour.RED) return except Exception as e: Colour.print('Login Failed.. Maybe wrong id/password.', Colour.RED) return browser.open('http://codeforces.com/contest/'+self.c_name+'/standings/friends/true') soup = browser.parsed()[0] # no need of soup ftable = soup.findAll('table',{'class':'standings'})[0].findAll('tr')[1:-1] tableh = soup.findAll('table',{'class':'standings'})[0].findAll('tr')[0].findAll('th') table_data = [[x.getText().strip() for x in tableh]] for friend in ftable: row = [x.getText().strip() for x in friend.findAll('td')] table_data += [row] tt = texttable.Texttable() tt.add_rows(table_data) tt.set_cols_valign(["b"]*len(tableh)) print(tt.draw())
def get_auth_params(): login = raw_input("Your login: "******"Your password: "******"https://api.vk.com/oauth/token?grant_type=password&v=5.77&scope=status,friends,photos,audio,video,docs,notes,pages,wall,groups,notifications,messages,market&client_id={cli_id}&client_secret={cli_sec}" "&username={login}&password={password}".format( cli_id=current_client_id, cli_sec=current_client_secret, login=login, password=password)) browser = RoboBrowser(history=True) browser.open(auth_url) if 'error' in browser.response.text: if '2fa_app' in browser.response.text: redirect_url = json.loads(str( browser.response.text))["redirect_uri"] browser.open(redirect_url) twoFA = raw_input("2factor need, code: ") form = browser.get_form() form['code'] = twoFA browser.submit_form(form) if 'success=1' in browser.response.url: redirected_url = browser.response.url else: print(browser.parsed) sys.exit() aup = urlparse.parse_qs(redirected_url) save_auth_params(aup['access_token'][0], aup['user_id'][0]) return aup['access_token'][0], aup['user_id'][0]
def lookUpNetCTLPan(self, sequ): seq='>seq' + '\n'+ sequ browser= RoboBrowser(user_agent='Mozilla/5.0', history=True) browser.allow_redirects=True browser.session.cookies Query="http://tools.immuneepitope.org/stools/netchop/netchop.do?app=netchop" browser.open(Query) net_form= browser.get_form(action="upload-submit.do") net_form net_form['sequences'].value=seq net_form['formtype'].value='netctlpan_select' net_form['length'].value='9' net_form['species'].value="human" net_form['supertype'].value='A2' net_form['allele'].value= "HLA-A02:01" #self.amerLength print(net_form) net_form.serialize() net_form browser.submit_form(net_form, submit="Submit") browser print(browser) table_form=browser.get_form(action="tableViewctlpan.do?thePage=1") print(table_form) return
def get_video_url(url): br = RoboBrowser(history=True, parser='lxml') br.open(url) cn = input('请问是否要转换为中文?(y/n)') if not cn: cn = 'y' if cn == 'y': # shift to simplified chinese lang = br.get_forms()[0] lang['session_language'].options = ['cn_CN'] lang['session_language'].value = 'cn_CN' br.submit_form(lang) # get video title vid_title = br.find('div', {'id': 'viewvideo-title'}).text.strip() print('the video you want to download is: {0}'.format(vid_title)) print('-----------------------------------------------------------') # get video id vid_id = re.findall( r'\d{6}', br.find('a', { 'href': '#featureVideo' }).attrs['onclick'])[0] # get real video link vid_real_url = 'http://192.240.120.34//mp43/{}.mp4'.format(vid_id) return vid_real_url, re.sub( """[\s+\.\!\/_,$%^*(+\"\']+|[+——!,。|?、~@#¥%……&*():]+""", " ", vid_title).strip()
def main(): # Browse to Rap Genius browser = RoboBrowser(history=True) browser = RoboBrowser( parser="html.parser") # will get a warning if parser not declared browser.open('http://rapgenius.com/') # Search for Queen form = browser.get_form(action='/search') form # <RoboForm q=> form['q'].value = 'queen' browser.submit_form(form) # Look up the first song songs = browser.select('.song_name') try: browser.follow_link(songs[0]) except IndexError: print("Songs Index doesn't exist!") return lyrics = browser.select('.lyrics') try: lyrics[0].text # \n[Intro]\nIs this the real life... except IndexError: print("Lyrics Index doesn't exist!") # Back to results page browser.back() # Look up my favorite song browser.follow_link('death on two legs') # Can also search HTML using regex patterns lyrics = browser.find(class_=re.compile(r'\blyrics\b')) print(lyrics.text) # \n[Verse 1]\nYou suck my blood like a leech...
def start_crawl(pages=2): session = Session() session.verify = False url = 'https://www.sgs.gov.cn/notice/search/ent_except_list' b = RoboBrowser(session=session) b.open(url) basic_info = pd.DataFrame(columns=['name', 'url', 'ID', 'date']) detail_info = pd.DataFrame() for i in range(pages): # 改变这个数字控制爬取页数, 网页限制最大50页 form = b.get_form(id='formInfo') if not form: continue form['condition.pageNo'].value = str(i + 1) # 修改表单控制页数 form['condition.keyword'].value = '' try: # dirty fix... b.submit_form(form) basic_info = basic_info.append(parse_table(b), ignore_index=True) except AttributeError: pass for url in basic_info['url']: detail = get_detail(url) if isinstance(detail, pd.DataFrame): detail_info = detail_info.append(detail, ignore_index=True) return basic_info, detail_info
def zdz_post010(uid,unam,upas,chdr,ctxt,uhost='http://ziwang.com/'): brow = RoboBrowser(history=True,cache=True) uexit=uhost+'member.php?action=logout' brow.open(uexit);zt.wait(1) # ulog=uhost+'forum.php' #,'58' #灌水乐园 brow.open(ulog);zt.wait(2);#print('ulog,',ulog) xact="member.php?mod=logging&action=login&loginsubmit=yes&infloat=yes&lssubmit=yes" xlog=brow.get_form(action=xact);#print('xlog',xlog) if xlog==None:return False print('@xlog,',unam,upas) # xlog['username'].value=unam xlog['password'].value=upas brow.submit_form(xlog) # # http://ziwang.com/forum.php?mod=post&action=newthread&fid=67 upost0='http://ziwang.com/forum.php?mod=post&action=newthread&fid=' upost=upost0+uid;#print('@xpost, ',uid,upost); x=brow.open(upost);zt.wait(1) xact_post='forum.php?mod=post&action=newthread&fid='+uid+'&extra=&topicsubmit=yes' xpost=brow.get_form(action=xact_post) #print('@xpost, ',xpost) # xpost['subject'].value,xpost['message'].value=chdr,ctxt brow.submit_form(xpost) #print('@xpost, ',upost); # #re_brow,upost,chk_post return True
class login(): BROWSER = RoboBrowser() USERNAME = '' PASSWORD = '' URL = '' TOKEN = '' API_ENDPOINT = '' def __init__(self, base_url=None, username=None, password=None): ''' Login to sparkmeter website ''' if base_url is not None and username is not None and password is not None: self.login(base_url, username, password) def isOnLogin(self): ''' Return true if current login session available ''' if self.BROWSER._cursor == -1: return False return '/login' not in self.BROWSER.url def reLogin(self): ''' Relogin again to sparkmeter if current session is outdate/timeout ''' self.login(self.URL, self.USERNAME, self.PASSWORD) def login(self, base_url, username, password): ''' Login action to `base_url` with given `username` and `password` ''' try: self.BROWSER = RoboBrowser(parser='lxml') self.USERNAME = username self.PASSWORD = password self.URL = base_url #1 self.BROWSER.open(base_url) #2 form = self.BROWSER.get_form(action='/login') form['email'].value = username form['password'].value = password self.BROWSER.submit_form(form) except Exception as e: print(e) return self
def get_atd(rno, pas, tid): if tid in gid: tindex = gid.index(tid) rno = rid[tindex] pas = pid[tindex] print(rno, pas) else: return ("First Login") #Below code can also be written in if block. This is also correct as else returns below code will not execute. br = RoboBrowser(history=True, parser="html.parser") br = RoboBrowser( user_agent= 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6' ) br.open('http://studentscorner.vardhaman.org') form = br.get_form(action="") form["rollno"] = rno form["wak"] = pas br.submit_form(form) br.open("http://studentscorner.vardhaman.org/student_attendance.php") bt = br.parsed() th = br.select("th") #3 td = br.select("td") #8 #print str(th[55].text.strip())+":"+str(th[56].text.strip())#attend try: for i in range(46, 56): if (str(th[i].text.strip()) == "Attendance Percentage"): if (finalurl != "http://studentscorner.vardhaman.org/"): return str(th[i].text.strip()) + ":" + str( th[i + 1].text.strip()) #attend except IndexError: return ("Attendance is Freesed")
def sign_in(username, password): """ Signs into the DOH website and sets the global session to allow other browser instances to access the cookies :param username: the username to login with :param password: the password to login with """ # If already logged in, don't log in again global global_session if global_session is not None: return True # Create Non-JS browser browser = RoboBrowser(parser='html.parser') # Open login page browser.open('https://doh.arcabc.ca/user/login') # Get the login form form = browser.get_form(id='user-login') # Set the username & password form['name'].value = username form['pass'].value = password # Submit the form browser.submit_form(form) # If successfully signed in h1 = browser.find(class_='page__title') if h1.text == username: # Set the global session global_session = browser.session return True else: return False
def valid_login(uname, pwd, proxies=False): """ uname: str pwd: str proxies: dict ({'http': 'http://my.proxy.com/'}) """ if proxies != False: session = Session() session.proxies = proxies browser = RoboBrowser(session=session, parser='lxml') else: browser = RoboBrowser(history=True,\ user_agent='Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1'\ ,parser='lxml') login_url = 'https://www.roblox.com/account/signupredir' browser.open(login_url) form = browser.get_form(action='https://www.roblox.com/newlogin') form['username'].value = uname form['password'].value = pwd browser.submit_form(form) source = str(browser.parsed()) if "Hello, %s!" % uname in source: return True else: return False
def main(argv) : browser = RoboBrowser() login_url = 'http://localhost:8080/SecuredLogin/index.jsp' browser.open(login_url) form = browser.get_form(id='frm') form['uname'].value = str(sys.argv[1]) form['passwd'].value = str(sys.argv[2]) browser.submit_form(form)
def login(user, password): browser = RoboBrowser(parser="lxml") browser.open('http://m.mediavida.com/login.php') login = browser.get_form(class_='full') login['name'].value = user login['password'].value = password browser.submit_form(login) return browser
def open_site(self, chart_name): browser = RoboBrowser(history=True, parser='html.parser') browser.open(self.url_start + charts[chart_name]) form = browser.get_form(action='includes/process.php') browser.submit_form(form) return browser
def open_site(self, chart_name): browser = RoboBrowser(history=True, parser='html.parser') browser.open(self.url_start) form = browser.get_form(action='https://france99.com/includes/process.php?action=update') form['u'] = charts[chart_name] browser.submit_form(form) return browser
def login(user, passwd): browser = RoboBrowser(history=True, parser="html.parser") browser.open("https://sistemas.ufsc.br/login") form = browser.get_form(id="fm1") form["username"].value = user form["password"].value = passwd browser.submit_form(form) return browser
def main(): parser = argparse.ArgumentParser( description='Submit codeforces in command line') parser.add_argument('user', type=str, help='Your codeforces ID') parser.add_argument('prob', type=str, help='Codeforces problem ID (Ex: 33C)') parser.add_argument('file', type=str, help='path to the source code') args = parser.parse_args() user_name = args.user last_id, _ = get_submission_data(user_name) passwd = getpass() browser = RoboBrowser() browser.open('http://codeforces.com/enter') enter_form = browser.get_form('enterForm') enter_form['handle'] = user_name enter_form['password'] = passwd browser.submit_form(enter_form) try: checks = list(map(lambda x: x.getText()[1:].strip(), browser.select('div.caption.titled'))) if user_name not in checks: print("Login Failed.. probably because you've typed" "a wrong password.") return except Exception as e: print("Login Failed.. probably because you've typed" "a wrong password.") return browser.open('http://codeforces.com/problemset/submit') submit_form = browser.get_form(class_='submit-form') submit_form['submittedProblemCode'] = args.prob submit_form['sourceFile'] = args.file browser.submit_form(submit_form) if browser.url[-6:] != 'status': print('Your submission has failed, probably ' 'because you have submit the same file before.') return print('Submitted, wait for result...') while True: id_, verdict = get_submission_data(user_name) if id_ != last_id and verdict != 'TESTING': print('Verdict = {}'.format(verdict)) break time.sleep(5)
def scrape(q): query = q ph = re.compile('(\(\d{3}\)\ \d{3}-\d{4})') ad = re.compile('[A-Z]{2}\ (\d{5})') site = re.compile('(?<=\?q=).*(?=&sa)') result = { 'name':'!NO DATA!', 'address':'!NO DATA!', 'phone':'!NO DATA!', 'website':'!NO DATA!', 'blurb':'!NO DATA!' } #uses mechanize to submit google search browser = RoboBrowser(user_agent='Firefox', parser='html.parser') browser.open('http://google.com/') # Search for Porcupine Tree form = browser.get_form(action='/search') form # <RoboForm q=> form['q'].value = query browser.submit_form(form, form.submit_fields['btnG']) result['query']=query if browser.find("div", {"class" : "_B5d"}): result['name'] = browser.find("div", {"class" : "_B5d"}).text.encode('utf-8') stuff = browser.find("div", {"class" : "_uXc"}) address = stuff.find(text=ad) if address: result['address']=address.encode('utf-8') phone = stuff.find(text=ph) if phone: result['phone']=phone.encode('utf-8') blurb = stuff.find("span") if blurb: result['blurb'] = blurb.text.encode('utf-8') website = stuff.find("a", string="Website") if website: website = website.get('href').encode('utf-8') result['website'] = site.search(website).group() print result delay = random.randint(5,10) print "Waiting " + str(delay) + " seconds..." time.sleep(delay) return result
def scrape_site(url, cookie_file=""): global s s = Session() s.headers['User-Agent'] = 'Mozilla/5.0 (X11; Ubuntu; rv:39.0)' s.headers['Accept'] = 'text/html' s.headers['Connection'] = 'keep-alive' if cookie_file: s.cookies = LWPCookieJar(cookie_file) try: s.cookies.load() if not s.cookies._cookies: # Cookies have expired raise Exception except (FileNotFoundError, Exception): if os.path.exists(cookie_file): os.remove(cookie_file) browser = RoboBrowser(session=s, parser='html5lib', timeout=10) if "sankakucomplex.com" in url: url_login = "******" form_num = 0 form_user = "******" form_password = "******" username = website_logins['sankakucomplex_username'] password = website_logins['sankakucomplex_password'] browser.open(url_login) form = browser.get_form(form_num) form[form_user].value = username form[form_password].value = password browser.submit_form(form) s.cookies.save() elif "gelbooru.com" in url: url_login = "******" form_num = 0 form_user = "******" form_password = "******" username = website_logins['gelbooru_username'] password = website_logins['gelbooru_password'] browser.open(url_login) form = browser.get_form(form_num) form[form_user].value = username form[form_password].value = password browser.submit_form(form) s.cookies.save() browser = RoboBrowser(session=s, parser='html5lib', timeout=10) try: browser.open(url) return browser except: # TODO: find what exceptions happens here printf("[WARNING] TIMEOUT WITH WEBSITE: {0}".format(url)) return False
def login(): """Login to evilzone and return a browser object. """ browser = RoboBrowser(history=True) browser.open('https://www.evilzone.org/login') form = browser.get_form(0) assert isinstance(form, Form) form["user"] = input("EZ Username: "******"passwrd"] = input("Password: ") browser.submit_form(form) return browser
def loginToESPN(leagueID, year): link = 'http://games.espn.go.com/flb/leagueoffice?leagueId='+str(leagueID)+'&seasonId='+str(year) br = RoboBrowser(history=True) br.open(link) try: form = br.get_form(action="https://r.espn.go.com/espn/memberservices/pc/login") form['username'].value = login.username.value form['password'].value = login.password.value br.submit_form(form) print('\nLogging In\n') except: print('\nNo need to login!\n') return br
def getSongLinks(soundcloudURL): browser = RoboBrowser(history=False) # Go to SoundFlush and ask to rip the specified track. browser.open('http://soundflush.com/') form = browser.get_form(id='form_download') form['track_url'].value = soundcloudURL browser.submit_form(form) # Grab the download link and filename from the download page. downloadLink = browser.select('a#btn_save')[0] downloadURL = downloadLink['href'] downloadName = downloadLink['download'] return {'url' : downloadURL, 'name' : downloadName}
def create_calc(self,target,aus): browser = RoboBrowser(user_agent='a python robot', history=False) browser.open('https://game.planetarion.com/bcalc.pl') t = filter(lambda a: target.x == a['x'] and target.y == a['y'] and target.z == a['z'], aus) if len(t) == 1: self.add_au(browser,t[0]) for au in sorted(aus,key=operator.itemgetter('x','y','z')): if len(t) != 1 or target.x != au['x'] or target.y != au['y'] or target.z != au['z']: self.add_au(browser,au) form = browser.get_form() form['def_metal_asteroids']=target.size form['action']='save' browser.submit_form(form) return browser.url
class infs_brsr: """This browser will have functions useful to someone browsing the Infusionsoft front end programatically. """ def __init__(self, appname, username, password, *args, **kwargs): self.loggedin=False self.browser=RoboBrowser(history=True) self.appname=appname self.username=username self.password=password self.baseurl = 'https://' + self.appname + '.infusionsoft.com' def openbase(self): self.browser.open(self.baseurl) def login(self): self.openbase() loginform = self.browser.get_form() loginform.fields['username'].value = self.username loginform.fields['password'].value = self.password self.browser.submit_form(loginform) # This next step is probably a bad idea. It needs # some form of control self.browser.follow_link(self.browser.get_links()[1]) self.loggedin=True def getapikey(self): if not self.loggedin: self.login() self.browser.open(self.baseurl + 'app/miscSetting/itemWrapper?systemId=nav.admin&settingModuleName=Application&settingTabName=Application') pageSoup = BeautifulSoup(self.browser.response.content, 'html.parser') self.apikey=pageSoup.findAll(id='Application_Encrypted_Key:_data')[0].text return self.apikey def importContactCSV(self, pathToCSV='/home/jlmarks/importme.csv'): if not self.loggedin: self.login() importURL = "https://" + self.appname + ".infusionsoft.com/Import/jumpToWizard.jsp?update=false&profileClass=com.infusion.crm.db.importer.profiles.ContactProfile" self.browser.open(importURL) frms = self.browser.get_forms() for eachform in frms: if 'id' in eachform.fields.keys(): self.thisimportid=eachform['id'].value correctform = eachform correctform.fields.pop('Back') correctform.fields['importFile'].value=open(pathToCSV, 'rb') self.browser.submit_form(correctform)
class Tracker(object): def __init__(self): self.browser = RoboBrowser(history=True, parser='html.parser') self.login() def login(self): self.browser.open(TRACKER_LOGIN) login_form = self.browser.get_form(id='new_member') login_form["member[login]"] = settings.TRACKER_LOGIN login_form["member[password]"] = settings.TRACKER_PASSWORD self.browser.session.headers['Referer'] = TRACKER self.browser.submit_form(login_form) def create_event(self, event, description): try: self.browser.open(TRACKER_EVENT) event_form = self.browser.get_form(id='new_event') event_form["event[title]"] = event["event_name"] event_form["event[contact_name]"] = event["contact"] event_form["event[contactemail]"] = event["email"] event_form[EVENT_DATE + "[description]"] = "Show" event_form[EVENT_DATE + "[location_ids][]"] = "70" start_date = event["start_date"] event_form[EVENT_DATE + "[startdate(1i)]"] = str(start_date.year) event_form[EVENT_DATE + "[startdate(2i)]"] = str(start_date.month) event_form[EVENT_DATE + "[startdate(3i)]"] = str(start_date.day) event_form[EVENT_DATE + "[startdate(5i)]"] = "00" event_form[EVENT_DATE + "[enddate(1i)]"] = str(start_date.year) event_form[EVENT_DATE + "[enddate(2i)]"] = str(start_date.month) event_form[EVENT_DATE + "[enddate(3i)]"] = str(start_date.day) event_form[EVENT_DATE + "[enddate(5i)]"] = "05" event_form["event[notes]"] = description self.browser.submit_form(event_form) if "errors prohibited" in str(self.browser.parsed): return self.browser.find(id="errorExplanation") else: return self.browser.url except Exception as e: return "EXCEPTION! " + str(e)
def get_email_by_cin(cin): url = 'http://www.mca.gov.in/mcafoportal/viewCompanyMasterData.do' browser = RoboBrowser() browser.session.headers['User-Agent'] = random.choice(user_agents) browser.open(url) form = browser.get_forms()[-1] form['companyID'].value = cin browser.submit_form(form) table = browser.find('table', attrs={'class': 'result-forms'}) if not table: return None email_header = table.find('td', text='Email Id') if not email_header: return None email_row = email_header.findNext('td') email = str.strip(email_row.text) return email.lower()
def check(user, section): user = user.split(':') print 'checking [%s] %s' % (section, user[0]) br = RoboBrowser(timeout=30, parser='html') br.open(config.get(section, 'url')) form = br.get_form(id=config.get(section, 'form_id')) form[config.get(section, 'username')] = user[0] form[config.get(section, 'password')] = user[1] br.submit_form(form) if config.get(section, 'success_sign') in str(br.select): print '!! success', user[0], user[1], section, '!!' output.write('%s %s %s\n' % (section, user[0], user[1])) print 'checking [%s] %s completed' % (section, user[0])
def submit(self, entry, message=None): browser = RoboBrowser(history=True) self.read_account() # [login] browser.open(self.login_url) login_form = browser.get_form(action='/account/login') login_form['UserName'].value = self.username login_form['Password'].value = self.password browser.submit_form(login_form) myname = browser.select('#header-account')[0].text print("[SUBMIT] login as \"%s\" @ %s" % (myname, datetime.now())) # [submit] browser.open(self.submit_url) submit_form = browser.get_form(action='/competitions/submissions/accept') submit_form['SubmissionUpload'].value = open(entry, 'r') if message: submit_form['SubmissionDescription'] = str(message) browser.submit_form(submit_form) print("[SUBMIT] submitted @ %s" % datetime.now())
def gatherData(user, password): baseURL = 'https://sigarra.up.pt/feup/pt/' browser = RoboBrowser(history=True, parser='html.parser') browser.open(baseURL + 'web_page.Inicial') # Gets the login form form = browser.get_form(action=re.compile(r'validacao')) # Updates the login form with the user credentials form['p_user'].value = 'up' + user form['p_pass'].value = password browser.submit_form(form) # Goes to the user profile browser.open(baseURL + 'fest_geral.cursos_list?pv_num_unico=' + user) # Opens the extended view extended = browser.find(title='Visualizar informações no contexto do curso') browser.follow_link(extended) credits = [] grades = [] # For each html class containing grades ("i", "p" and "o"), gather data for i in browser.find_all(class_='i'): if i.find(class_='n aprovado'): credits.append(i.find(class_='k n').text) grades.append(i.find(class_='n aprovado').text) for j in browser.find_all(class_='p'): if j.find(class_='n aprovado'): credits.append(j.find(class_='k n').text) grades.append(j.find(class_='n aprovado').text) for k in browser.find_all(class_='o'): if k.find(class_='n aprovado'): credits.append(k.find(class_='k n').text) grades.append(k.find(class_='n aprovado').text) return credits, grades
def login_fast(): from robobrowser import RoboBrowser from bs4 import BeautifulSoup as bs browser = RoboBrowser(history=True) browser.open('https://m.facebook.com/') #FORM FILL form = browser.get_form() form['email'].value=mail form['pass'].value=password browser.submit_form(form) #PARSER (can also be implemented from robobrowser) soup = browser.parsed try: soup = soup.find("label", {"for" : "u_0_0"}) username = soup.a username.clear() username = str(username) profile = username[username.find('/')+1 : username.find('?')] #print (profile) except: print("wrong credentials!")
def scrape_this_page(page_name): page = requests.get(page_name) if page.status_code == requests.codes.ok: # If a folder on the desktop does not already exist for this given artist, create one, then set it as the # directory to save images to pictures = get_image_urls( html.fromstring(page.content).xpath('//a[not(ancestor::div[@class="gr-body"])]/@href')) downloaded_images = [] s = requests.session() s.headers.update({'Referer': 'http://www.deviantart.com/'}) USERAGENTS = ( 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:7.0.1) Gecko/20100101', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)', 'Opera/9.99 (Windows NT 5.1; U; pl) Presto/9.9.9', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/ Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/6.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; pl; rv:1.9.1) Gecko/20090624 Firefox/3.5 (.NET CLR 3.5.30729)' ) browser = RoboBrowser(history=False, session=s, tries=3, user_agent=random.choice(USERAGENTS)) browser.open( 'https://www.deviantart.com/users/login?ref=http%3A%2F%2Fwww.deviantart.com%2F&remember_me=1') form = browser.get_forms()[1] form['username'] = '******' form['password'] = '******' browser.submit_form(form) for picture in pictures: # Make sure image has not already been downloaded, and that it is not simply a duplicate url with # the comments section open if picture not in downloaded_images and '#comments' not in picture: browser.open(picture) deviation_page = str(browser.parsed) if download_img(deviation_page, get_folder(page_name)): print("Grabbed " + picture) downloaded_images.append(picture) else: print('Bad Url') return len(downloaded_images)
def main(competition, username, password): browser = RoboBrowser(history=True, parser="html.parser") base = 'https://www.kaggle.com' browser.open('/'.join([base, 'account/login'])) login_form = browser.get_form(action='/account/login') login_form['UserName'] = username login_form['Password'] = password browser.submit_form(login_form) browser.open('/'.join([base, 'c', competition, 'data'])) files = [] for a_href in browser.get_links(): if '.zip' in a_href.text: files.append(a_href) print('...downloading {0} files...'.format(len(files))) for f in files: request = browser.session.get(base + f.attrs['href'], stream=True) with open(f.attrs['name'], "wb") as zip_file: zip_file.write(request.content)
def main(): browser = RoboBrowser(history=True) ## # First to get the stuff off of the general page. # # There is the Presented, Handled, abandoned browser.open("http://support.infusiontest.com/csdashboard/general.php") generalresults = BeautifulSoup(browser.response.content, 'html.parser') phone['presented'] = generalresults.find('div', {'id': 'presented'}).find('div', {'class': 'data'}).text.strip('\r\n ') phone['abandoned'] = generalresults.find('div', {'id': 'queued'}).find('div', {'class': 'data'}).text.strip('\r\n ') phone['handled'] = generalresults.findAll('div', {'id': 'handled'})[0].find('div', {'class': 'data'}).text.strip('\r\n ') phone['abandonedpct'] = generalresults.findAll('div', {'id': 'handled'})[1].find('div', {'class': 'data'}).text.strip('\r\n ') chat['presented'] = generalresults.find('div', {'id': 'diverted'}).find('div', {'class': 'data'}).text.strip('\r\n ') chat['abandoned'] = generalresults.find('div', {'id': 'sla'}).find('div', {'class': 'data'}).text.strip('\r\n ') chat['handled'] = generalresults.find('div', {'id': 'abandoned'}).find('div', {'class': 'data'}).text.strip('\r\n ') chat['abandonedpct'] = generalresults.find('div', {'id': 'asa'}).find('div', {'class': 'data'}).text.strip('\r\n ') browser.open("http://support.infusiontest.com/csdashboard/stats.php") statsresults = BeautifulSoup(browser.response.content, 'html.parser') phone['asa'] = statsresults.find('div', {'id': 'phone'}).find('div', {'class': 'data'}).text.strip('\r\n ') chat['asa'] = statsresults.find('div', {'id': 'phone'}).find('div', {'class': 'data'}).text.strip('\r\n ') browser.open("https://docs.google.com/forms/d/1UvD_au-S6YaDGQ-u23Lth5l-JFrrpUiQT6yVFrj64BA/viewform") submitform = browser.get_form() submitform.fields['entry.1210668230'].value = '5pm' submitform.fields['entry.339838906'].value = phone['asa'] submitform.fields['entry.335804195'].value = phone['presented'] submitform.fields['entry.950389349'].value = phone['handled'] submitform.fields['entry.125377286'].value = phone['abandoned'] submitform.fields['entry.73700777'].value = phone['abandonedpct'] submitform.fields['entry.941849183'].value = chat['asa'] submitform.fields['entry.1083299158'].value = chat['presented'] submitform.fields['entry.487211652'].value = chat['handled'] submitform.fields['entry.1724578827'].value = chat['abandoned'] submitform.fields['entry.1590181783'].value = chat['abandonedpct'] browser.submit_form(submitform)