def support(x_cord, y_cord, browser): branfrage = mechanize.Request( "http://de101.die-staemme.de/game.php?village=5512&screen=place") response = browser.open(branfrage) forms = ParseResponse(response) form = forms[0] #print form control = form.find_control(name="support", type="submit") #print control.name, control.value, control.type form["sword"] = "180" form["x"] = str(x_cord) form["y"] = str(y_cord) oeffnen = form.click(control.name) antwort = browser.open(oeffnen) forms2 = ParseResponse(antwort) form2 = forms2[0] #print form2 control2 = form2.find_control(type="submit") #print control2.name, control2.value, control2.type oeffnen = form2.click(control2.type) browser.open(oeffnen)
def farmsearch(browser): globals.lock.acquire(blocking=1) conn = sqlite3.connect('tribal.db') c = conn.cursor() c.execute("drop table farm") conn.commit() c.execute("create table farm(id integer primary key, x int, y int)") conn.commit for i in range(544, 562): for j in range(486, 502): try: print("ich suche gerade nach", i, "|", j) branfrage = mechanize.Request( "http://de101.die-staemme.de/game.php?village=5512&screen=place" ) browser.open(branfrage) response = browser.open(branfrage) forms = ParseResponse(response) form = forms[0] #print form control = form.find_control(name="support", type="submit") #print control.name, control.value, control.type form["sword"] = "1" form["x"] = str(i) form["y"] = str(j) oeffnen = form.click(control.name) antwort = browser.open(oeffnen) forms2 = ParseResponse(antwort) form2 = forms2[0] try: vergleich = form2.find_control(name="support", type="submit") except: player = False for link in browser.links(): if "id" in link.url and "info_player" in link.url: player = True if player is False: print i, "|", j c.execute("insert into farm(x,y) values (?,?)", [i, j]) except Exception, e: print e print("fehler") time.sleep(random.uniform(1, 10))
def connect(url, username, password): try: if connected(url): raise LinkException('You are already connected') try: response = urlopen(url) except URLError: raise SSIDException('You are not connected on a FON box') forms = ParseResponse(response, backwards_compat=False) try: form = forms[0] form["login[USERNAME]"] = username form["login[PASSWORD]"] = password except IndexError: raise SSIDException('You are not connected on a FON box') try: response_page = urlopen(form.click()).read() except NameError: raise SSIDException('You are not connected on a FON box') return not 'class="form_error"' in response_page except PlainURLError: if connected(url): return True else: raise RuntimeError("Connection failed.")
def login(self, username, password): response = urlopen(urljoin(self.uri, "/?cs=login")) forms = ParseResponse(response, backwards_compat=False) form = forms[0] form.set_value(username, name='username') form.set_value(password, name='password') self.page = urlopen(form.click())
def main(self): keyword = remove_polish(word) openable = 1 response = urlopen(uri) forms = ParseResponse(response, backwards_compat=False) if len(forms)==0: os.system("python PyCrawler.py"+" baza.db '"+ uri+"' 1 "+ keyword) return form = forms[0] # search for text input in form and put keyword there control = form.find_control(type="text") control.value = keyword # form.click() returns a mechanize.Request object # (see HTMLForm.click.__doc__ if you want to use only the forms support, and # not the rest of mechanize) request2 = form.click() # mechanize.Request object try: response2 = urlopen(request2) except: print "Nie mozna otworzyc formularza" openable = 0 pass #get the url of page if not openable: search_url=uri else: search_url = response2.geturl() #start crawler on it os.system("python PyCrawler.py"+" baza.db '"+ search_url+"' 1 "+ keyword)
def attack(village, spear, sword, axe, spy, light, heavy, ram, catapult, snob, x_cord,y_cord,browser): globals.lock.acquire(blocking=1) link = linkbuilder.linkbuilder(village,'attack') branfrage = mechanize.Request(link) response = browser.open(branfrage) forms = ParseResponse(response) form = forms[0] control = form.find_control(name = "attack", type="submit") form["spear"] = spear form["sword"] = sword form["axe"] = axe form["spy"] = spy form["light"] = light form["heavy"] = heavy form["ram"] = ram form["catapult"] = catapult form["snob"] = snob form["x"] = x_cord form["y"] = y_cord oeffnen = form.click(control.name) antwort = browser.open(oeffnen) forms2 = ParseResponse(antwort) form2 = forms2[0] control2 = form2.find_control( type="submit") oeffnen = form2.click(control2.type) browser.open(oeffnen) globals.lock.release()
def get_bclerk_results_text(self, case): uri = 'http://web1.brevardclerk.us/oncoreweb/search.aspx' response = urllib.urlopen(uri) forms = ParseResponse(response, backwards_compat=False) form = forms[0] # print form form["txtCaseNumber"] = case # "orozco" form["SearchType"] = 'casenumber' # "orozco" form["txtDocTypes"] = '' # 'JRP, J' #"orozco" # form["txtName"] = "orozco" # time.sleep(1) bclerk_results_text = urlopen(form.click()).read() return bclerk_results_text
def get_rows_from_graframe_url(url3, radius): # print('get_rows_from_graframe_url(%s, %s)' % (url3, radius)) r3 = urlopen(url3) forms = ParseResponse(r3, backwards_compat=False) # for f in forms: # print f form = forms[0] # print(form) form["select"] = ['Sales'] form["radius"] = [radius] # print(form) rows = get_nearby_from_input(urlopen(form.click())) # pprint.pprint(rows) return rows
def getURLFormInformation(): #Small function to craw a web page and print all information about forms on the page, for debug purposes #Open URL response = urlopen(LOGIN_URL) #Obtain form information forms = ParseResponse(response, backwards_compat=False) form = forms[0] print form #Add values to form form['email'] = USERNAME form['pass'] = USERNAME print form['email'], form['pass'] os.system('pause')
def getShowsList(self): #returns dict of {id, show} and keys forms = ParseResponse(self.page, backwards_compat=False) selectList = forms[1] control = selectList.controls[0] listValues = [item.attrs['value'] for item in control.items] listLabels = [item.attrs['label'] for item in control.items] #right now this works, but I think it is just lucky -- there may be sorting issues showList = {} i = 0 for item in listValues: showList[item] = listLabels[i] i += 1 keys = listValues return showList, keys
import scraperwiki import lxml.html from mechanize import ParseResponse, urlopen, urljoin uri = "http://www.censusindia.gov.in/Census_Data_2001/Census_Data_Online/Area_Profile/SubDistrict_Profile.aspx?cki=AoDoRSqLMhi" response = urlopen(uri) #print response.read() forms = ParseResponse(response, backwards_compat=False) #print forms form = forms[0] #print form statecode = [] serial = 1 statename = [] #for item in form.find_control("drpState").items: # statecode.append(item.name) # statename.append(item.attrs['contents']) #print statename #print statecode #statecount=0 #for i in statecode: # if(i==""): # statecount+=1 # continue # else: districtcode = [] districtname = [] form.set_value(['28'], name="drpState") content = urlopen(form.click())
def getformpage(self, user, password, choice): projectgroupurl = "http://leo.rp.edu.sg//workspace/studentModule.asp?site=" qnnurl = "http://leo3.rp.edu.sg//projectweb/group_evaluation.asp?" quizurl = "http://leo3.rp.edu.sg//projectweb/qnn_take.asp?" sessionurl = "http://leo3.rp.edu.sg//projectweb/qnn_preset.asp?" charturl = "http://leo3.rp.edu.sg//projectweb/response_chart.asp?" saveurl = "http://leo3.rp.edu.sg//projectweb/qnn_save_responses.asp" urllist = [] for i in range(1, 4): urllist.append(projectgroupurl + str(i)) # retrieve the result currentModule = "projectid" currentProblem = "groupid" try: for url in urllist: opener = self.addAuthentication(url, user, password) mechanize.install_opener(opener) response = mechanize.Request(url) page = urlopen(response).read() if ("Wrong Password" in page or "Wrong ID" in page): self.vNP.set("Sorry, USERNAME or PASSWORD wrong!") elif ('''ToggleDisplay''' in page): currentModule = self.getidlist("projectid", page)[-1] currentProblem = self.getidlist("groupid", page)[-1] if (currentModule != "projectid" and currentProblem != "groupid"): getqnnurl = qnnurl + currentModule + "&" + currentProblem + "&lang=ISO-8859-1" opener = self.addAuthentication(getqnnurl, user, password) mechanize.install_opener(opener) response = mechanize.Request(getqnnurl) getqnnpage = urlopen(response) forms = ParseResponse(getqnnpage, backwards_compat=False) form = forms[0] qnnid = form["qnnid"] evalid = form["evalid"] opener = self.addAuthentication(getqnnurl, user, password) mechanize.install_opener(opener) response = mechanize.Request(getqnnurl) getqnnpageread = urlopen(response).read() author_evaluatorlist = re.findall(r"'\d{5}', '.{38}'", getqnnpageread) #for i in range(len(author_evaluatorlist)): authorid = author_evaluatorlist[0][1:6] evaluatorid = author_evaluatorlist[0][10:-1] getsessionurl = sessionurl + "&qnnid=" + qnnid + "&" + currentModule + "&" + currentProblem + "&evalid=" + evalid + "&evaltype=P" + "&authorid=" + authorid + "&evaluatorid=" + evaluatorid + "&lang=ISO-8859-1" opener = self.addAuthentication(getsessionurl, user, password) mechanize.install_opener(opener) response = mechanize.Request(getsessionurl) getqnnpage = urlopen(response) forms = ParseResponse(getqnnpage, backwards_compat=False) form = forms[0] form.set_all_readonly(False) form["qnnid"] = qnnid form["authorid"] = authorid form["evaluatorid"] = evaluatorid form["evaltype"] = "P" form["lang"] = "ISO-8859-1" form["newflag"] = "0" form["evalid"] = evalid form["groupid"] = currentProblem[8:] form["projectid"] = currentModule[10:] submit = form.click() data = submit.get_data() opener = self.addAuthentication(quizurl, user, password) mechanize.install_opener(opener) response = mechanize.Request(quizurl, data) sessionid = self.getidlist("sessionid", urlopen(response).read())[0] answerurl = re.search( "(\<FRAME NAME=\"main\" SRC=\")(.+)(\"\>)", urlopen(response).read()).group(2) answerurl = "http://leo3.rp.edu.sg//projectweb/" + answerurl opener = self.addAuthentication(answerurl, user, password) mechanize.install_opener(opener) rs = mechanize.Request(answerurl, data) quiz = urlopen(rs) quizpage = urlopen(rs).read() questionid = re.search(r"\{.+\}num", quizpage).group()[0:-3] forms = ParseResponse(quiz, backwards_compat=False) form = forms[0] self.fillform(form, choice, questionid, sessionid, charturl, user, password) form.set_all_readonly(False) form["finish"] = "MANUAL" print form ''' data = form.click().get_data() opener = self.addAuthentication(saveurl, user, password) mechanize.install_opener(opener) req = mechanize.Request(saveurl, data) req.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6') req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8') req.add_header('Accept-Encoding', 'gzip,deflate') print urlopen(req).read() ''' else: self.vNP.set("Sorry, TODAY NO MODULE!") except mechanize.HTTPError, e: self.vNP.set( "Error:", BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code])
def farmb(list,browser): conn = sqlite3.connect('tribal.db') globals.lock.acquire(blocking=1) link = linkbuilder.linkbuilder(1, 'attack') tmp = 0 light = True #antwort = mechanize.urlopen(anfrage) for i in range(len(list)): try: #print("Ich bin bei Angriff ",i+1) c = conn.cursor() c.execute('select x,y from farm where id = (?)', [list[i]]) coord = [dict(x=row[0], y=row[1]) for row in c.fetchall()] xcord = coord[0]['x'] ycord = coord[0]['y'] branfrage = mechanize.Request(link) response = browser.open(branfrage) forms = ParseResponse(response) form = forms[0] #print form control = form.find_control(name = "attack", type="submit") #print control.name, control.value, control.type ''' <TextControl(spear=)> <TextControl(sword=)> <TextControl(axe=)> <TextControl(spy=)> <TextControl(light=)> <TextControl(heavy=)> <TextControl(ram=)> <TextControl(catapult=)> <TextControl(snob=)>''' if light == True: form["light"] = "10" '''elif axe == True: form["axe"] = "10"''' form["x"] = str(xcord) form["y"] = str(ycord) time.sleep(random.uniform(0,1)) oeffnen = form.click(control.name) antwort = browser.open(oeffnen) forms2 = ParseResponse(antwort) form2 = forms2[0] #print form2 control2 = form2.find_control(type="submit") #print control2.name, control2.value, control2.type oeffnen2 = form2.click(control2.type) browser.open(oeffnen2) time.sleep(random.uniform(1,5)) except: if tmp == 0: link = linkbuilder.linkbuilder(3,'attack') tmp = 1 elif tmp == 1: link = linkbuilder.linkbuilder(4,'attack') tmp = 2 else: i = len(list)+1 conn.close() globals.lock.release()