def support(x_cord, y_cord, browser):
branfrage = mechanize.Request(
"http://de101.die-staemme.de/game.php?village=5512&screen=place")
response = browser.open(branfrage)
forms = ParseResponse(response)
form = forms[0]
#print form
control = form.find_control(name="support", type="submit")
#print control.name, control.value, control.type
form["sword"] = "180"
form["x"] = str(x_cord)
form["y"] = str(y_cord)
oeffnen = form.click(control.name)
antwort = browser.open(oeffnen)
forms2 = ParseResponse(antwort)
form2 = forms2[0]
#print form2
control2 = form2.find_control(type="submit")
#print control2.name, control2.value, control2.type
oeffnen = form2.click(control2.type)
browser.open(oeffnen)
def farmsearch(browser):
globals.lock.acquire(blocking=1)
conn = sqlite3.connect('tribal.db')
c = conn.cursor()
c.execute("drop table farm")
conn.commit()
c.execute("create table farm(id integer primary key, x int, y int)")
conn.commit
for i in range(544, 562):
for j in range(486, 502):
try:
print("ich suche gerade nach", i, "|", j)
branfrage = mechanize.Request(
"http://de101.die-staemme.de/game.php?village=5512&screen=place"
)
browser.open(branfrage)
response = browser.open(branfrage)
forms = ParseResponse(response)
form = forms[0]
#print form
control = form.find_control(name="support", type="submit")
#print control.name, control.value, control.type
form["sword"] = "1"
form["x"] = str(i)
form["y"] = str(j)
oeffnen = form.click(control.name)
antwort = browser.open(oeffnen)
forms2 = ParseResponse(antwort)
form2 = forms2[0]
try:
vergleich = form2.find_control(name="support",
type="submit")
except:
player = False
for link in browser.links():
if "id" in link.url and "info_player" in link.url:
player = True
if player is False:
print i, "|", j
c.execute("insert into farm(x,y) values (?,?)", [i, j])
except Exception, e:
print e
print("fehler")
time.sleep(random.uniform(1, 10))
def connect(url, username, password):
try:
if connected(url):
raise LinkException('You are already connected')
try:
response = urlopen(url)
except URLError:
raise SSIDException('You are not connected on a FON box')
forms = ParseResponse(response, backwards_compat=False)
try:
form = forms[0]
form["login[USERNAME]"] = username
form["login[PASSWORD]"] = password
except IndexError:
raise SSIDException('You are not connected on a FON box')
try:
response_page = urlopen(form.click()).read()
except NameError:
raise SSIDException('You are not connected on a FON box')
return not 'class="form_error"' in response_page
except PlainURLError:
if connected(url):
return True
else:
raise RuntimeError("Connection failed.")
def login(self, username, password):
response = urlopen(urljoin(self.uri, "/?cs=login"))
forms = ParseResponse(response, backwards_compat=False)
form = forms[0]
form.set_value(username, name='username')
form.set_value(password, name='password')
self.page = urlopen(form.click())
def main(self):
keyword = remove_polish(word)
openable = 1
response = urlopen(uri)
forms = ParseResponse(response, backwards_compat=False)
if len(forms)==0:
os.system("python PyCrawler.py"+" baza.db '"+ uri+"' 1 "+ keyword)
return
form = forms[0]
# search for text input in form and put keyword there
control = form.find_control(type="text")
control.value = keyword
# form.click() returns a mechanize.Request object
# (see HTMLForm.click.__doc__ if you want to use only the forms support, and
# not the rest of mechanize)
request2 = form.click() # mechanize.Request object
try:
response2 = urlopen(request2)
except:
print "Nie mozna otworzyc formularza"
openable = 0
pass
#get the url of page
if not openable:
search_url=uri
else:
search_url = response2.geturl()
#start crawler on it
os.system("python PyCrawler.py"+" baza.db '"+ search_url+"' 1 "+ keyword)
def attack(village, spear, sword, axe, spy, light, heavy, ram, catapult, snob, x_cord,y_cord,browser):
globals.lock.acquire(blocking=1)
link = linkbuilder.linkbuilder(village,'attack')
branfrage = mechanize.Request(link)
response = browser.open(branfrage)
forms = ParseResponse(response)
form = forms[0]
control = form.find_control(name = "attack", type="submit")
form["spear"] = spear
form["sword"] = sword
form["axe"] = axe
form["spy"] = spy
form["light"] = light
form["heavy"] = heavy
form["ram"] = ram
form["catapult"] = catapult
form["snob"] = snob
form["x"] = x_cord
form["y"] = y_cord
oeffnen = form.click(control.name)
antwort = browser.open(oeffnen)
forms2 = ParseResponse(antwort)
form2 = forms2[0]
control2 = form2.find_control( type="submit")
oeffnen = form2.click(control2.type)
browser.open(oeffnen)
globals.lock.release()
def get_bclerk_results_text(self, case):
uri = 'http://web1.brevardclerk.us/oncoreweb/search.aspx'
response = urllib.urlopen(uri)
forms = ParseResponse(response, backwards_compat=False)
form = forms[0]
# print form
form["txtCaseNumber"] = case # "orozco"
form["SearchType"] = 'casenumber' # "orozco"
form["txtDocTypes"] = '' # 'JRP, J' #"orozco"
# form["txtName"] = "orozco"
# time.sleep(1)
bclerk_results_text = urlopen(form.click()).read()
return bclerk_results_text
def get_rows_from_graframe_url(url3, radius):
# print('get_rows_from_graframe_url(%s, %s)' % (url3, radius))
r3 = urlopen(url3)
forms = ParseResponse(r3, backwards_compat=False)
# for f in forms:
# print f
form = forms[0]
# print(form)
form["select"] = ['Sales']
form["radius"] = [radius]
# print(form)
rows = get_nearby_from_input(urlopen(form.click()))
# pprint.pprint(rows)
return rows
def getURLFormInformation():
#Small function to craw a web page and print all information about forms on the page, for debug purposes
#Open URL
response = urlopen(LOGIN_URL)
#Obtain form information
forms = ParseResponse(response, backwards_compat=False)
form = forms[0]
print form
#Add values to form
form['email'] = USERNAME
form['pass'] = USERNAME
print form['email'], form['pass']
os.system('pause')
def getShowsList(self):
#returns dict of {id, show} and keys
forms = ParseResponse(self.page, backwards_compat=False)
selectList = forms[1]
control = selectList.controls[0]
listValues = [item.attrs['value'] for item in control.items]
listLabels = [item.attrs['label'] for item in control.items]
#right now this works, but I think it is just lucky -- there may be sorting issues
showList = {}
i = 0
for item in listValues:
showList[item] = listLabels[i]
i += 1
keys = listValues
return showList, keys
import scraperwiki
import lxml.html
from mechanize import ParseResponse, urlopen, urljoin
uri = "http://www.censusindia.gov.in/Census_Data_2001/Census_Data_Online/Area_Profile/SubDistrict_Profile.aspx?cki=AoDoRSqLMhi"
response = urlopen(uri)
#print response.read()
forms = ParseResponse(response, backwards_compat=False)
#print forms
form = forms[0]
#print form
statecode = []
serial = 1
statename = []
#for item in form.find_control("drpState").items:
# statecode.append(item.name)
# statename.append(item.attrs['contents'])
#print statename
#print statecode
#statecount=0
#for i in statecode:
# if(i==""):
# statecount+=1
# continue
# else:
districtcode = []
districtname = []
form.set_value(['28'], name="drpState")
content = urlopen(form.click())
def getformpage(self, user, password, choice):
projectgroupurl = "http://leo.rp.edu.sg//workspace/studentModule.asp?site="
qnnurl = "http://leo3.rp.edu.sg//projectweb/group_evaluation.asp?"
quizurl = "http://leo3.rp.edu.sg//projectweb/qnn_take.asp?"
sessionurl = "http://leo3.rp.edu.sg//projectweb/qnn_preset.asp?"
charturl = "http://leo3.rp.edu.sg//projectweb/response_chart.asp?"
saveurl = "http://leo3.rp.edu.sg//projectweb/qnn_save_responses.asp"
urllist = []
for i in range(1, 4):
urllist.append(projectgroupurl + str(i))
# retrieve the result
currentModule = "projectid"
currentProblem = "groupid"
try:
for url in urllist:
opener = self.addAuthentication(url, user, password)
mechanize.install_opener(opener)
response = mechanize.Request(url)
page = urlopen(response).read()
if ("Wrong Password" in page or "Wrong ID" in page):
self.vNP.set("Sorry, USERNAME or PASSWORD wrong!")
elif ('''ToggleDisplay''' in page):
currentModule = self.getidlist("projectid", page)[-1]
currentProblem = self.getidlist("groupid", page)[-1]
if (currentModule != "projectid" and currentProblem != "groupid"):
getqnnurl = qnnurl + currentModule + "&" + currentProblem + "&lang=ISO-8859-1"
opener = self.addAuthentication(getqnnurl, user, password)
mechanize.install_opener(opener)
response = mechanize.Request(getqnnurl)
getqnnpage = urlopen(response)
forms = ParseResponse(getqnnpage, backwards_compat=False)
form = forms[0]
qnnid = form["qnnid"]
evalid = form["evalid"]
opener = self.addAuthentication(getqnnurl, user, password)
mechanize.install_opener(opener)
response = mechanize.Request(getqnnurl)
getqnnpageread = urlopen(response).read()
author_evaluatorlist = re.findall(r"'\d{5}', '.{38}'",
getqnnpageread)
#for i in range(len(author_evaluatorlist)):
authorid = author_evaluatorlist[0][1:6]
evaluatorid = author_evaluatorlist[0][10:-1]
getsessionurl = sessionurl + "&qnnid=" + qnnid + "&" + currentModule + "&" + currentProblem + "&evalid=" + evalid + "&evaltype=P" + "&authorid=" + authorid + "&evaluatorid=" + evaluatorid + "&lang=ISO-8859-1"
opener = self.addAuthentication(getsessionurl, user, password)
mechanize.install_opener(opener)
response = mechanize.Request(getsessionurl)
getqnnpage = urlopen(response)
forms = ParseResponse(getqnnpage, backwards_compat=False)
form = forms[0]
form.set_all_readonly(False)
form["qnnid"] = qnnid
form["authorid"] = authorid
form["evaluatorid"] = evaluatorid
form["evaltype"] = "P"
form["lang"] = "ISO-8859-1"
form["newflag"] = "0"
form["evalid"] = evalid
form["groupid"] = currentProblem[8:]
form["projectid"] = currentModule[10:]
submit = form.click()
data = submit.get_data()
opener = self.addAuthentication(quizurl, user, password)
mechanize.install_opener(opener)
response = mechanize.Request(quizurl, data)
sessionid = self.getidlist("sessionid",
urlopen(response).read())[0]
answerurl = re.search(
"(\<FRAME NAME=\"main\" SRC=\")(.+)(\"\>)",
urlopen(response).read()).group(2)
answerurl = "http://leo3.rp.edu.sg//projectweb/" + answerurl
opener = self.addAuthentication(answerurl, user, password)
mechanize.install_opener(opener)
rs = mechanize.Request(answerurl, data)
quiz = urlopen(rs)
quizpage = urlopen(rs).read()
questionid = re.search(r"\{.+\}num", quizpage).group()[0:-3]
forms = ParseResponse(quiz, backwards_compat=False)
form = forms[0]
self.fillform(form, choice, questionid, sessionid, charturl,
user, password)
form.set_all_readonly(False)
form["finish"] = "MANUAL"
print form
'''
data = form.click().get_data()
opener = self.addAuthentication(saveurl, user, password)
mechanize.install_opener(opener)
req = mechanize.Request(saveurl, data)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6')
req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
req.add_header('Accept-Encoding', 'gzip,deflate')
print urlopen(req).read()
'''
else:
self.vNP.set("Sorry, TODAY NO MODULE!")
except mechanize.HTTPError, e:
self.vNP.set(
"Error:",
BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code])
def farmb(list,browser):
conn = sqlite3.connect('tribal.db')
globals.lock.acquire(blocking=1)
link = linkbuilder.linkbuilder(1, 'attack')
tmp = 0
light = True
#antwort = mechanize.urlopen(anfrage)
for i in range(len(list)):
try:
#print("Ich bin bei Angriff ",i+1)
c = conn.cursor()
c.execute('select x,y from farm where id = (?)', [list[i]])
coord = [dict(x=row[0], y=row[1]) for row in c.fetchall()]
xcord = coord[0]['x']
ycord = coord[0]['y']
branfrage = mechanize.Request(link)
response = browser.open(branfrage)
forms = ParseResponse(response)
form = forms[0]
#print form
control = form.find_control(name = "attack", type="submit")
#print control.name, control.value, control.type
''' <TextControl(spear=)>
<TextControl(sword=)>
<TextControl(axe=)>
<TextControl(spy=)>
<TextControl(light=)>
<TextControl(heavy=)>
<TextControl(ram=)>
<TextControl(catapult=)>
<TextControl(snob=)>'''
if light == True:
form["light"] = "10"
'''elif axe == True:
form["axe"] = "10"'''
form["x"] = str(xcord)
form["y"] = str(ycord)
time.sleep(random.uniform(0,1))
oeffnen = form.click(control.name)
antwort = browser.open(oeffnen)
forms2 = ParseResponse(antwort)
form2 = forms2[0]
#print form2
control2 = form2.find_control(type="submit")
#print control2.name, control2.value, control2.type
oeffnen2 = form2.click(control2.type)
browser.open(oeffnen2)
time.sleep(random.uniform(1,5))
except:
if tmp == 0:
link = linkbuilder.linkbuilder(3,'attack')
tmp = 1
elif tmp == 1:
link = linkbuilder.linkbuilder(4,'attack')
tmp = 2
else:
i = len(list)+1
conn.close()
globals.lock.release()