def login(self, username, password):
response = urlopen(urljoin(self.uri, "/?cs=login"))
forms = ParseResponse(response, backwards_compat=False)
form = forms[0]
form.set_value(username, name='username')
form.set_value(password, name='password')
self.page = urlopen(form.click())
def login(self, username, password):
response = urlopen(urljoin(self.uri, "/?cs=login"))
forms = ParseResponse(response, backwards_compat=False)
form = forms[0]
form.set_value(username, name='username')
form.set_value(password, name='password')
self.page = urlopen(form.click())
def clean_url(self, url, response=None):
"""
string URL -- convert to absolute URL
"""
if response:
base_url = response.geturl()
else:
base_url = self.base_url
# use mechanize.urljoin
return mechanize.urljoin(base_url, url)
def randWord(wordType, wordComplexity): randURL = 'http://watchout4snakes.com' response = urlopen(urljoin(randURL, '/wo4snakes/Random/RandomWordPlus')) forms = ParseResponse(response, backwards_compat=False) form = forms[0] form['Pos'] = [wordType] form['Level'] = [wordComplexity] return urlopen(form.click()).read()
def load_courses(self):
profile_link = "/explorer/profile"
for link in self.br.links():
if link.url == profile_link:
break
link.absolute_url = mechanize.urljoin(self.base_url, link.url)
self.br.follow_link(link)
# Scrapping urls to user's courses
profilePage = BeautifulSoup(self.br.response().read(), 'html.parser')
course_tags = profilePage.find_all('th', {'class':'gcb-align-left'})
links_dict = {}
for links in self.br.links():
if links.text != None:
links_dict[(links.text).strip()] = links
for course in course_tags[2:]:
dict = {}
link_tag = course.find('a')
key = (link_tag.text).strip()
dict['Name'] = key
dict['URL'] = links_dict[key]
dict['URL'].absolute_url = mechanize.urljoin(self.base_url, dict['URL'].url)
self.course_links.append(dict)
def connect_to_form(url, formurl):
print "\nConnecting to Web Page...",
# Connect to URL (add error handling!!)
request = mechanize.Request(mechanize.urljoin(url, formurl))
response = mechanize.urlopen(request)
monkeypatch_mechanize()
print "Success."
# Retrieve forms
forms = mechanize.ParseResponse(response, backwards_compat=False)
response.close()
if len(forms) <= 0:
raise FormNotFound('No Forms were found on the web page.')
return forms
def connect_to_form(url, formurl):
print "\nConnecting to Web Page...",
# Connect to URL (add error handling!!)
request = mechanize.Request(mechanize.urljoin(url, formurl))
response = mechanize.urlopen(request)
monkeypatch_mechanize()
print "Success."
# Retrieve forms
forms = mechanize.ParseResponse(response, backwards_compat=False)
response.close()
if len(forms) <= 0:
raise FormNotFound('No Forms were found on the web page.')
return forms
def getIncidents():
uri = "http://umnprd.service-now.com/"
br = Browser()
br.set_handle_robots(False)
response1 = br.open(uri)
br.select_form("lform")
br["j_username"] = sys.argv[1]
br["j_password"] = sys.argv[2]
br.submit()
br.select_form(nr=0)
br.submit()
response3 = br.open(urljoin(uri, "incident_list.do?XML&sysparm_query=active=true^assignment_groupLIKECLA%20East%20Bank"))
incidents_xml = str(response3.get_data())
incidents = parseString(incidents_xml)
return incidents
def find_torrent(br, query, se):
rsp = br.open(url + "/torrents/browse/index/query/" + query + "/categories/26")
soup = BeautifulSoup(rsp, "html.parser")
dl = soup.find("table", {"id": "torrenttable"})
for link in dl.find_all("a"):
if link.get("href").endswith(".torrent"):
dl = link.get("href")
if ("SPANISH" not in dl.upper()) and ("HEBSUB" not in dl.upper()):
if findEPNum(dl, se):
urldl = urljoin(url, dl)
fname = dl.split("/")[-1]
torrent = br.open(urldl).read()
file = open("/path/to/downloads/" + fname, "w")
file.write(torrent)
file.close()
break
else:
continue
def login_redmine(br,username="",pwd = ""):
"""
登陆redmine
@param br :当前的浏览器对象
username:登录用的用户名
pwd :登录用的密码
@type br :mechanize.Browser
username:str
pwd :str
@return none
@rtype none
"""
response = br.open(mechanize.urljoin(cfg.get("redmine","REDMINE_HOST"),"login"))
br.select_form(nr=0)
br.form["username"] = (username and username) or cfg.get("redmine","USER_NAME")
br.form["password"] = (pwd and pwd) or cfg.get("redmine","USER_PWD")
login_response = br.submit()
write_to(login_response.read(),"login.html")
def to_import(br, porject_key=None, file_path=None):
"""
进入的上传文件页，提交上传文件
@param br :当前的浏览器对象
porject_key:需要执行操作的project的key
file_path :上传文件的路径
@type br :mechanize.Browser
porject_key :str
file_path :str
@return none
@rtype none
"""
if not porject_key or not file_path:
print "no param of porject_key or upload file path ,which project[file] you want to operate?"
return
print cfg.get("redmine", "REDMINE_HOST"), porject_key
response = br.open(
mechanize.urljoin(cfg.get("redmine", "REDMINE_HOST"),
"importer?project_id=" + porject_key))
real_url = response.geturl()
if real_url.rfind("/login?") > 0:
print "loginout ... "
return False
br.select_form(nr=1) #选择上传文件的表单
upload_form = br.form
#enc_control = upload_form.find_control("encoding",type="select")
#enc_control.get("U").selected = True
#upload_form["splitter"] =","
#upload_form["wrapper"] ="\""
upload_form.add_file(open(file_path, "rb"),
"application/vnd.ms-excel",
file_path,
name="file")
#print upload_form
upload_response = br.submit()
write_to(upload_response.read(), "upload.html")
return True
def book_ticket(self, tickets, people, code=None):
print "Booking tickets for %s people..." % len(people)
print "Selecting tickets%s..." % ' and applying discount code' if code else ""
booking_url = self.get_booking_url()
(reg_info_fields, reqd_fields) = self._get_reg_info_field_ids(reqd=True)
self.browser.open(booking_url)
self.browser.select_form(nr=0)
tickets_data = dict(("qty_" + ticket, dict(data=str(len(people)))) for ticket in tickets)
tickets_map = dict(
inputs=dict(code='code'),
listcontrols=dict((ticket_key, ticket_key) for ticket_key in tickets_data))
if code:
tickets_data['code'] = dict(data=code)
form = self._fill_form(self.browser.form, tickets_data, tickets_map)
self.browser.open(form.click())
if self.browser.geturl() == urljoin(booking_url, 'orders/addons'):
self.browser.select_form(nr=0)
self.browser.open(self.browser.form.click())
self.browser.select_form(nr=0)
extra_fields = ['company', 'jobtitle', 'phone', 'city', 'twitter', 'tshirt']
form = self._init_form(self.browser.form)
for i, person in enumerate(people):
print "Updating information for %s..." % person['name']
form.set_value(person['name'], name='order[participation_attributes][][name]', nr=i)
form.set_value(person['email'], name='order[participation_attributes][][email]', nr=i)
for j, field in enumerate(reg_info_fields):
# This will fail for T-shirts as it is of listcontrol type.
# In our current use case T-shirt info is never going to be updated.
# In case we update T-shirt info in future, we should use _fill_form and provide a custom mapper dict to it. E.g. tickets_map above
if extra_fields[j] in person and person[extra_fields[j]]:
form['info_%s_%s' % (i, field)] = person[extra_fields[j]]
elif field in reqd_fields:
form['info_%s_%s' % (i, field)] = "*"
for ticket in tickets:
form['terms_' + ticket] = ['true']
self.browser.open(form.click())
print "Tickets booked..."
print self.browser.geturl()
def to_import(br,porject_key = None,file_path = None):
"""
进入的上传文件页，提交上传文件
@param br :当前的浏览器对象
porject_key:需要执行操作的project的key
file_path :上传文件的路径
@type br :mechanize.Browser
porject_key :str
file_path :str
@return none
@rtype none
"""
if not porject_key or not file_path:
print "no param of porject_key or upload file path ,which project[file] you want to operate?"
return
print cfg.get("redmine","REDMINE_HOST"),porject_key
response = br.open(mechanize.urljoin(cfg.get("redmine","REDMINE_HOST"),"importer?project_id="+porject_key))
real_url = response.geturl()
if real_url.rfind("/login?")>0:
print "loginout ... "
return False
br.select_form(nr=1) #选择上传文件的表单
upload_form = br.form
#enc_control = upload_form.find_control("encoding",type="select")
#enc_control.get("U").selected = True
#upload_form["splitter"] =","
#upload_form["wrapper"] ="\""
upload_form.add_file(open(file_path,"rb"),"application/vnd.ms-excel",file_path,name="file")
#print upload_form
upload_response = br.submit()
write_to(upload_response.read(),"upload.html")
return True
import sys
import mechanize
if len(sys.argv) == 1:
uri = "https://750words.com/auth"
else:
uri = sys.argv[1]
request = mechanize.Request(mechanize.urljoin(uri, ''))
response = mechanize.urlopen(request)
forms = mechanize.ParseResponse(response, backwards_compat=False)
response.close()
## f = open("example.html")
## forms = mechanize.ParseFile(f, "http://example.com/example.html",
## backwards_compat=False)
## f.close()
form = forms[0]
print form # very useful!
#!/usr/bin/env python
import sys
import mechanize
if len(sys.argv) == 1:
uri = "http://wwwsearch.sourceforge.net/"
else:
uri = sys.argv[1]
request = mechanize.Request(mechanize.urljoin(uri, "mechanize/example.html"))
response = mechanize.urlopen(request)
forms = mechanize.ParseResponse(response, backwards_compat=False)
response.close()
## f = open("example.html")
# forms = mechanize.ParseFile(f, "http://example.com/example.html",
# backwards_compat=False)
# f.close()
form = forms[0]
print form # very useful!
# A 'control' is a graphical HTML form widget: a text entry box, a
# dropdown 'select' list, a checkbox, etc.
# Indexing allows setting and retrieval of control values
original_text = form["comments"] # a string, NOT a Control instance
form["comments"] = "Blah."
# Controls that represent lists (checkbox, select and radio lists) are
# ListControl instances. Their values are sequences of list item names.
#!/usr/bin/env python
import sys
import mechanize
if len(sys.argv) == 1:
uri = "http://wwwsearch.sourceforge.net/"
else:
uri = sys.argv[1]
br = mechanize.Browser()
br.open(mechanize.urljoin(uri, "mechanize/example.html"))
forms = list(br.forms())
# f = open("example.html")
# forms = mechanize.ParseFile(f, "http://example.com/example.html",
# backwards_compat=False)
# f.close()
form = forms[0]
print form # very useful!
# A 'control' is a graphical HTML form widget: a text entry box, a
# dropdown 'select' list, a checkbox, etc.
# Indexing allows setting and retrieval of control values
original_text = form["comments"] # a string, NOT a Control instance
form["comments"] = "Blah."
# Controls that represent lists (checkbox, select and radio lists) are
# ListControl instances. Their values are sequences of list item names.
# They come in two flavours: single- and multiple-selection:
import mechanize
from mechanize import ParseResponse, urlopen, urljoin
import re
import scraperwiki
import string
base = "http://www.dleg.state.mi.us/bcs_corp/"
search_page = "sr_corp.asp"
result_page = "dt_corp.asp"
# Next page link: rs_corp.asp?s_button=sname&v_search=a&hiddenField=&offset=40
# Get the main name search form
print urljoin(base, search_page)
main_page = urlopen(urljoin(base, search_page))
br = mechanize.Browser()
br.open(base)
forms = mechanize.ParseResponse(main_page, backwards_compat=False)
form = forms[0]
print form
# Search for something:
form.set_value("a", name="v_search")
br.open(form.click())
# Find all URLS that begin with 'dt_corp.asp'
# for letter in string.lowercase:
# print letter
# Import module statements
import re, sys, httplib, urllib
from bs4 import BeautifulSoup
from mechanize import ParseResponse, urlopen, urljoin
# To read the information about a device, you will need the proper device ID. Enter device ID from the web URL in place of XXXXXXXXXX.
if len(sys.argv) == 1:
uri = "https://graph.api.smartthings.com/device/show/XXXXXXXXXX"
else:
uri = sys.argv[1]
# Open URL and parse out information to see what forms are required to login
response = urlopen(urljoin(uri, ""))
forms = ParseResponse(response, backwards_compat=False)
form = forms[0]
# Uncomment line below to print out the form names used for logging in
#print form
# Enter information for username and password. Must put information in quotes.
form["j_username"] = "******"
form["j_password"] = "******"
# Send login information and read webpage data
text_data = urlopen(form.click()).read()
# START DATA PARSING
# Get HTML code of web page to parse through
soup = BeautifulSoup(text_data)
# Find line that has desired data. This case grabs the line with temperature.
text = soup.find_all(text=re.compile("temperature"))
def sync_event(event):
syncform = EventSyncForm()
if request.method == 'GET' or not syncform.validate_on_submit():
return render_template('form.html', form=syncform, title=u"Sync", submit=u"Sync Now")
else:
browser = Browser(factory=RobustFactory())
speakers = []
speakers_fetched = False
lastuser_loggedin = False
eventframe_data = None
ret = []
if (event.funnel_space or event.eventframe_sync) and request.form.get('lastuser_username', None) and request.form.get('lastuser_password', None):
browser.open("https://auth.hasgeek.com/login")
browser.select_form(nr=1)
form = browser.form
form['username'] = request.form.get('lastuser_username')
form['password'] = request.form.get('lastuser_password')
browser.open(form.click())
if browser.geturl() == "https://auth.hasgeek.com/login":
flash("Problem logging into LastUser", "danger")
return render_template('form.html', form=EventSyncForm(), title=u"Sync", submit=u"Sync Now")
else:
lastuser_loggedin = True
if lastuser_loggedin and event.funnel_space:
browser.open("https://funnel.hasgeek.com/login")
ret.append("Fetching speakers")
browser.open("https://funnel.hasgeek.com/{space}/json".format(space=event.funnel_space))
funnel_data = json.loads(browser.response().read())
for proposal in funnel_data['proposals']:
if proposal['confirmed']:
proposal['speaker'] = unicode(proposal['speaker'])
if u'(' in proposal['speaker']:
proposal['speaker'] = u'('.join(proposal['speaker'].split('(')[:-1]).strip().title()
speakers.append((proposal['speaker'], proposal['email']))
speakers = list(set(speakers))
speakers_fetched = True
ret.append("There are %s speakers" % len(speakers))
ret.append("Syncing venues from Funnel")
try:
names = [venue['name'] for venue in funnel_data['venues']]
for venue in Venue.query.filter_by(event=event, from_funnel=True).filter(~Venue.name.in_(names)).all():
for activity in venue.activity:
db.session.delete(activity)
db.session.delete(venue)
db.session.commit()
ret.append("Deleted removed venues")
except Exception as e:
ret.append("Error deleting removed venues: %s" % str(e))
venues = dict()
for venue in funnel_data['venues']:
try:
venues[venue['name']] = Venue.query.filter_by(name=venue['name'], event=event, from_funnel=True).one()
ret.append("Venue %s exists as %s" % (venue['title'], venues[venue['name']].title))
except NoResultFound:
try:
venues[venue['name']] = Venue(event=event, name=venue['name'], title=venue['title'], from_funnel=True)
db.session.add(venues[venue['name']])
db.session.commit()
ret.append("Added venue %s" % venue['title'])
except Exception as e:
ret.append("Error adding venue %s: %s" % (venue['title'], str(e)))
for room in funnel_data['rooms']:
venues[room['name']] = venues[room['venue']]
activity = defaultdict(list)
for day_number, day in enumerate(funnel_data['schedule']):
for slot in day['slots']:
for session in slot['sessions']:
if session['room'] and session['room'] in venues:
if (day_number, day['date']) not in activity[session['room']]:
activity[session['room']].append((day_number, day['date']))
for venue, days in activity.iteritems():
try:
dates = [date for (day_number, date) in days]
Activity.query.filter_by(venue=venues[venue], from_funnel=True).filter(~Activity.date.in_(dates)).delete(synchronize_session=False)
ret.append("Deleted removed activity days for %s" % venues[venue].title)
except Exception as e:
ret.append("Error deleting removed activity days for %s: %s" % (venues[venue].title, str(e)))
for day_number, date in days:
try:
item = Activity.query.filter_by(venue=venues[venue], date=date, from_funnel=True).one()
ret.append("Activity on %s exists as %s" % (item.date, item.title))
except NoResultFound:
try:
item = Activity(venue=venues[venue], date=date, title="Day %s - %s" % (day_number + 1, venues[venue].title), from_funnel=True)
db.session.add(item)
db.session.commit()
ret.append("Added activity: %s on %s" % (item.title, item.date))
except Exception as e:
ret.append("Error adding activity %s: %s" % ("Day %s - %s" % (day_number + 1, venues[venue].title), str(e)))
ret.append("Funnel sync complete")
if lastuser_loggedin and event.eventframe_sync:
ret.append("Fetching data from Eventframe")
browser.open("https://eventframe.hasgeek.com/login")
browser.open(event.eventframe_sync)
eventframe_data = json.loads(browser.response().read())['attendees']
ret.append("Fetched %s people from Eventframe" % len(eventframe_data))
if app.config['DOATTEND_EMAIL'] in [None, ''] or app.config['DOATTEND_PASS'] in [None, ''] or event.doattend_id in [None, '']:
return 'Data not available'
uri = 'http://doattend.com/'
forms = ParseResponse(urlopen(urljoin(uri, 'accounts/sign_in')))
form = forms[0]
form['account[email]'] = app.config['DOATTEND_EMAIL']
form['account[password]'] = app.config['DOATTEND_PASS']
urlopen(form.click())
csv_data = urlopen(urljoin(uri, 'events/' + event.doattend_id + '/orders/registration_sheet.csv')).read()
guests_csv = urlopen(urljoin(uri, 'events/' + event.doattend_id + '/orders/confirmed_guests.csv')).read()
urlopen(urljoin(uri, 'accounts/sign_out'))
f = StringIO.StringIO(csv_data)
headers = [make_name(field).replace(u'-', u'').replace(u'\n', u'') for field in f.next().split(',')]
users = unicodecsv.reader(f, delimiter=',')
added = [0]
failed = []
updated = [0]
deleted = 0
tickets = []
ret.append("Starting Participants")
def process_ticket(user):
user[columns['name']] = user[columns['name']].title()
participant = Participant.query.filter_by(ticket_number=user[columns['ticket_number']].strip(), event_id=event.id, online_reg=True).first()
if not participant:
for p in Participant.query.filter_by(email=user[columns['email']].strip(), event_id=event.id).all():
if levenshtein(p.name, user[columns['name']].strip()) <= 3:
participant = p
break
elif participant.email != user[columns['email']]:
participant.image = 'LOAD'
if not columns['name'] or user[columns['name']] == 'Cancelled':
return
new = participant is None
if new:
participant = Participant()
participant.event_id = event.id
participant.purchases = []
participant.image = 'LOAD'
else:
if participant.purchases is None:
participant.purchases = []
else:
participant.purchases = participant.purchases.split(', ')
if 'order_id' in columns and columns['order_id'] and user[columns['order_id']]:
user[columns['order_id']] = int(user[columns['order_id']])
if columns['twitter'] and user[columns['twitter']] and '@' in user[columns['twitter']]:
user[columns['twitter']] = user[columns['twitter']].strip().replace('@', '').strip()
if columns['phone'] and user[columns['phone']]:
user[columns['phone']] = user[columns['phone']].strip().replace(' ', '').replace('-','')
for field in columns.keys():
if columns[field]:
value = user[columns[field]]
if type(value) == unicode:
value = value.strip()
value = None if value == '*' or value == '' or value == 'empty' else value
if (new or (field != 'ticket_number' and getattr(participant, field))) and value != getattr(participant, field):
setattr(participant, field, value)
if speakers_fetched and participant.speaker and (participant.name, participant.email) not in speakers:
participant.speaker = False
if (participant.name, participant.email) in speakers:
participant.speaker = True
if user[others['ticket_type']]:
participant.purchases.append(user[others['ticket_type']].strip())
if 'addons' in others and others['addons'] and user[others['addons']]:
participant.purchases = participant.purchases + (user[others['addons']]).strip().split(',')
for i, purchase in enumerate(participant.purchases):
participant.purchases[i] = purchase.strip().replace(u'Apr 18 - 19', u'May 16 - 17').replace(u'Apr 16 - 17', u'May 14 - 15').replace(u'Apr 16 - 19', u'May 14 - 17')
if u"T-shirt" in participant.purchases or u"Corporate" in participant.purchases:
participant.purchased_tee = True
participant.purchases = ', '.join(list(set(participant.purchases)))
participant.online_reg = True
if participant.ticket_number:
tickets.append(participant.ticket_number)
try:
if new:
db.session.add(participant)
db.session.commit()
if new:
added[0] = added[0] + 1
ret.append("Added " + participant.name.encode('utf-8'))
else:
updated[0] = updated[0] + 1
ret.append("Updated " + participant.name.encode('utf-8'))
except Exception as e:
ret.append("Error adding/updating " + participant.name.encode('utf-8') + ': ' + str(e))
failed.append(participant.name.encode('utf-8') + '(' + participant.email.encode('utf-8') + '): ' + str(e))
db.session.rollback()
def indexof(name):
try:
return headers.index(name)
except ValueError:
return None
columns = dict(
ticket_number=indexof(u'ticketnumber'),
name=indexof(u'name'),
email=indexof(u'email'),
company=indexof(u'company'),
job=indexof(u'jobtitle'),
city=indexof(u'city'),
phone=indexof(u'phone'),
twitter=indexof(u'twitterhandle'),
regdate=indexof(u'date'),
order_id=indexof(u'orderid')
)
others = dict(
ticket_type=indexof(u'ticketname'),
addons=indexof(u'addonspurchased')
)
for user in users:
process_ticket(user)
ret.append("Done with Participants")
ret.append("Starting Guests")
f = StringIO.StringIO(guests_csv)
headers = [make_name(field).replace(u'-', u'').replace(u'\n', u'') for field in f.next().split(',')]
users = unicodecsv.reader(f, delimiter=',')
columns = dict(
ticket_number=indexof(u'ticketnumber'),
name=indexof(u'name'),
email=indexof(u'email'),
company=indexof(u'company'),
job=indexof(u'jobtitle'),
city=indexof(u'city'),
phone=indexof(u'phone'),
twitter=indexof(u'twitterhandle'),
regdate=indexof(u'confirmedon')
)
others = dict(
ticket_type=indexof(u'ticket')
)
for user in users:
process_ticket(user)
ret.append("Done with Guests")
if eventframe_data:
ret.append("Starting Eventframe RSVP data")
columns = dict(
ticket_number='ticket',
name='ticket_name',
email='ticket_email',
phone='ticket_phone',
company='ticket_company',
job='ticket_jobtitle',
city='ticket_city',
twitter='ticket_twitter'
)
others = dict(
ticket_type='ticket_type'
)
for user in eventframe_data:
if user['status'] == "Y" and 'ticket' in user:
process_ticket(user)
ret.append("Done with RSVPs")
ret.append("Removing deleted tickets")
participants = Participant.query.filter(~Participant.ticket_number.in_(tickets), Participant.online_reg==True, Participant.event_id==event.id).all()
for participant in participants:
try:
db.session.delete(participant)
db.session.commit()
deleted = deleted + 1
ret.append("Delete: " + participant.name.encode('utf-8'))
except Exception as e:
ret.append("Error deleting " + participant.name.encode('utf-8') + ':' + e)
ret.append("Deleting complete")
for participant in Participant.query.filter_by(event=event, image='LOAD').all():
ret.append("Loading gravatar image for %s" % ( participant.email))
try:
response = requests.get(
"http://www.gravatar.com/avatar/" + md5(participant.email.lower()).hexdigest(),
params={'d': '404', 's': '400'})
if response.status_code == 404:
participant.image = None
ret.append('Image not present')
else:
participant.image = b64encode(response.content)
ret.append('Image loaded: %s' % participant.image)
db.session.commit()
except Exception as e:
ret.append('Error: ' + str(e))
return json.dumps(dict(
added=added[0],
updated=updated[0],
deleted=deleted,
failed=failed,
trace=ret
))
import sys
import mechanize
if len(sys.argv) == 1:
uri = "https://750words.com/auth"
else:
uri = sys.argv[1]
request = mechanize.Request(mechanize.urljoin(uri, ''))
response = mechanize.urlopen(request)
forms = mechanize.ParseResponse(response, backwards_compat=False)
response.close()
## f = open("example.html")
## forms = mechanize.ParseFile(f, "http://example.com/example.html",
## backwards_compat=False)
## f.close()
form = forms[0]
print form # very useful!
import mechanize
from mechanize import ParseResponse, urlopen, urljoin
import re
import scraperwiki
import string
base = "http://www.dleg.state.mi.us/bcs_corp/"
search_page = "sr_corp.asp"
result_page = "dt_corp.asp"
# Next page link: rs_corp.asp?s_button=sname&v_search=a&hiddenField=&offset=40
# Get the main name search form
print urljoin(base, search_page)
main_page = urlopen(urljoin(base, search_page))
br = mechanize.Browser()
br.open(base)
forms = mechanize.ParseResponse(main_page, backwards_compat=False)
form = forms[0]
print form
# Search for something:
form.set_value("a", name="v_search")
br.open(form.click())
# Find all URLS that begin with 'dt_corp.asp'
Simple working example (examples/forms/simple.py in the source distribution):
import sys
from mechanize import ParseResponse, urlopen, urljoin
if len(sys.argv) == 1:
uri = "http://wwwsearch.sourceforge.net/"
else:
uri = sys.argv[1]
response = urlopen(urljoin(uri, "mechanize/example.html"))
forms = ParseResponse(response, backwards_compat=False)
form = forms[0]
print form
form["comments"] = "Thanks, Gisle"
# form.click() returns a mechanize.Request object
# (see HTMLForm.click.__doc__ if you want to use only the forms support, and
# not the rest of mechanize)
print urlopen(form.click()).read()
A more complicated working example (from examples/forms/example.py in the source distribution):
import sys
import mechanize
if len(sys.argv) == 1:
uri = "http://wwwsearch.sourceforge.net/"
else:
uri = sys.argv[1]
def sync_event(event):
syncform = EventSyncForm()
if request.method == 'GET' or not syncform.validate_on_submit():
return render_template('form.html',
form=syncform,
title=u"Sync",
submit=u"Sync Now")
else:
browser = Browser(factory=RobustFactory())
speakers = []
speakers_fetched = False
lastuser_loggedin = False
eventframe_data = None
ret = []
if (event.funnel_space or event.eventframe_sync) and request.form.get(
'lastuser_username', None) and request.form.get(
'lastuser_password', None):
browser.open("https://auth.hasgeek.com/login")
browser.select_form(nr=1)
form = browser.form
form['username'] = request.form.get('lastuser_username')
form['password'] = request.form.get('lastuser_password')
browser.open(form.click())
if browser.geturl() == "https://auth.hasgeek.com/login":
flash("Problem logging into LastUser", "danger")
return render_template('form.html',
form=EventSyncForm(),
title=u"Sync",
submit=u"Sync Now")
else:
lastuser_loggedin = True
if lastuser_loggedin and event.funnel_space:
browser.open("https://funnel.hasgeek.com/login")
ret.append("Fetching speakers")
browser.open("https://funnel.hasgeek.com/{space}/json".format(
space=event.funnel_space))
funnel_data = json.loads(browser.response().read())
for proposal in funnel_data['proposals']:
if proposal['confirmed']:
proposal['speaker'] = unicode(proposal['speaker'])
if u'(' in proposal['speaker']:
proposal['speaker'] = u'('.join(
proposal['speaker'].split(
'(')[:-1]).strip().title()
speakers.append(
(proposal['speaker'], proposal['email']))
speakers = list(set(speakers))
speakers_fetched = True
ret.append("There are %s speakers" % len(speakers))
ret.append("Syncing venues from Funnel")
try:
names = [venue['name'] for venue in funnel_data['venues']]
for venue in Venue.query.filter_by(
event=event, from_funnel=True).filter(
~Venue.name.in_(names)).all():
for activity in venue.activity:
db.session.delete(activity)
db.session.delete(venue)
db.session.commit()
ret.append("Deleted removed venues")
except Exception as e:
ret.append("Error deleting removed venues: %s" % str(e))
venues = dict()
for venue in funnel_data['venues']:
try:
venues[venue['name']] = Venue.query.filter_by(
name=venue['name'], event=event,
from_funnel=True).one()
ret.append(
"Venue %s exists as %s" %
(venue['title'], venues[venue['name']].title))
except NoResultFound:
try:
venues[venue['name']] = Venue(event=event,
name=venue['name'],
title=venue['title'],
from_funnel=True)
db.session.add(venues[venue['name']])
db.session.commit()
ret.append("Added venue %s" % venue['title'])
except Exception as e:
ret.append("Error adding venue %s: %s" %
(venue['title'], str(e)))
for room in funnel_data['rooms']:
venues[room['name']] = venues[room['venue']]
activity = defaultdict(list)
for day_number, day in enumerate(funnel_data['schedule']):
for slot in day['slots']:
for session in slot['sessions']:
if session['room'] and session['room'] in venues:
if (day_number, day['date']
) not in activity[session['room']]:
activity[session['room']].append(
(day_number, day['date']))
for venue, days in activity.iteritems():
try:
dates = [date for (day_number, date) in days]
Activity.query.filter_by(
venue=venues[venue], from_funnel=True).filter(
~Activity.date.in_(dates)).delete(
synchronize_session=False)
ret.append("Deleted removed activity days for %s" %
venues[venue].title)
except Exception as e:
ret.append(
"Error deleting removed activity days for %s: %s" %
(venues[venue].title, str(e)))
for day_number, date in days:
try:
item = Activity.query.filter_by(
venue=venues[venue],
date=date,
from_funnel=True).one()
ret.append("Activity on %s exists as %s" %
(item.date, item.title))
except NoResultFound:
try:
item = Activity(
venue=venues[venue],
date=date,
title="Day %s - %s" %
(day_number + 1, venues[venue].title),
from_funnel=True)
db.session.add(item)
db.session.commit()
ret.append("Added activity: %s on %s" %
(item.title, item.date))
except Exception as e:
ret.append(
"Error adding activity %s: %s" %
("Day %s - %s" %
(day_number + 1, venues[venue].title),
str(e)))
ret.append("Funnel sync complete")
if lastuser_loggedin and event.eventframe_sync:
ret.append("Fetching data from Eventframe")
browser.open("https://eventframe.hasgeek.com/login")
browser.open(event.eventframe_sync)
eventframe_data = json.loads(
browser.response().read())['attendees']
ret.append("Fetched %s people from Eventframe" %
len(eventframe_data))
added = [0]
failed = []
updated = [0]
deleted = 0
if app.config['DOATTEND_EMAIL'] in [
None, ''
] or app.config['DOATTEND_PASS'] in [
None, ''
] or event.doattend_id in [None, '']:
ret.append('DoAttend details not available')
else:
uri = 'http://doattend.com/'
browser.open(urljoin(uri, 'accounts/sign_in'))
browser.select_form(nr=0)
form = browser.form
form['account[email]'] = app.config['DOATTEND_EMAIL']
form['account[password]'] = app.config['DOATTEND_PASS']
browser.open(form.click())
if browser.geturl() == urljoin(uri, 'accounts/sign_in'):
ret.append('DoAttend Login Failed')
else:
ret.append('Syncing tickets')
tickets = []
browser.open(
urljoin(uri, 'events/' + event.doattend_id + '/tickets'))
resp = html.fromstring(browser.response().read())
tickets_list = resp.get_element_by_id('tickets_list')
tickets_list = tickets_list.cssselect('.list')
ticket_ids = []
for ticket in tickets_list:
tickets.append({
'id':
ticket.attrib['id'].split('_')[1],
'name':
ticket.cssselect('div:nth-child(2) h3')
[0].text.strip()
})
ticket_ids.append(ticket.attrib['id'].split('_')[1])
tickets_list = resp.cssselect('#tickets_list')[1]
tickets_list = tickets_list.cssselect('.list')
for ticket in tickets_list:
tickets.append({
'id':
ticket.attrib['id'].split('_')[1],
'name':
ticket.cssselect('div:nth-child(2) h3')
[0].text.strip()
})
ticket_ids.append(ticket.attrib['id'].split('_')[1])
try:
ret.append('Deleting removed tickets')
Product.query.filter_by(
event=event, source='doattend').filter(
~Product.id.in_(ticket_ids)).delete(
synchronize_session=False)
db.session.commit()
ret.append('Removed tickets deleted')
except Exception as e:
ret.append("Error deleting removed tickets: %s" % str(e))
for ticket in tickets:
try:
t = Product.query.filter_by(id_source=ticket['id'],
event=event,
source='doattend').one()
ret.append("Ticket %s(%s) exists as %s" %
(ticket['name'], ticket['id'], t.title))
except NoResultFound:
try:
t = Product(id_source=ticket['id'],
event=event,
title=ticket['name'],
source='doattend')
t.make_name()
db.session.add(t)
db.session.commit()
ret.append("Added ticket %s(%s)" %
(ticket['name'], ticket['id']))
except Exception as e:
ret.append("Error adding ticket %s: %s" %
(ticket['name'], str(e)))
db.session.commit()
browser.open(
urljoin(
uri, 'events/' + event.doattend_id +
'/orders/registration_sheet.csv'))
csv_data = browser.response().read()
browser.open(
urljoin(
uri, 'events/' + event.doattend_id +
'/orders/confirmed_guests.csv'))
guests_csv = browser.response().read()
browser.open(urljoin(uri, 'accounts/sign_out'))
tickets = []
f = StringIO.StringIO(csv_data)
headers = [
make_name(field).replace(u'-', u'').replace(u'\n', u'')
for field in f.next().split(',')
]
users = unicodecsv.reader(f, delimiter=',')
ret.append("Starting Participants")
def process_ticket(user):
user[columns['name']] = user[columns['name']].title()
participant = Participant.query.filter_by(
ticket_number=user[columns['ticket_number']].strip(),
event_id=event.id,
online_reg=True).first()
if not participant:
for p in Participant.query.filter_by(
email=user[columns['email']].strip(),
event_id=event.id).all():
if levenshtein(p.name,
user[columns['name']].strip()) <= 3:
participant = p
break
elif participant.email != user[columns['email']]:
participant.image = 'LOAD'
if not columns['name'] or user[
columns['name']] == 'Cancelled' or user[
columns['name']] == 'Not Attending':
return
new = participant is None
if new:
participant = Participant()
participant.event_id = event.id
participant.purchases = []
participant.image = 'LOAD'
else:
if participant.purchases is None:
participant.purchases = []
else:
participant.purchases = participant.purchases.split(
', ')
if 'order_id' in columns and columns['order_id'] and user[
columns['order_id']]:
user[columns['order_id']] = int(
user[columns['order_id']])
if columns['twitter'] and user[columns[
'twitter']] and '@' in user[columns['twitter']]:
user[columns['twitter']] = user[
columns['twitter']].strip().replace('@',
'').strip()
if columns['phone'] and user[columns['phone']]:
user[columns['phone']] = user[
columns['phone']].strip().replace(' ', '').replace(
'-', '')
for field in columns.keys():
if columns[field]:
value = user[columns[field]]
if type(value) == unicode:
value = value.strip()
value = None if value == '*' or value == '' or value == 'empty' else value
if (new or (field != 'ticket_number'
and getattr(participant, field))
) and value != getattr(participant, field):
setattr(participant, field, value)
if speakers_fetched and participant.speaker and (
participant.name,
participant.email) not in speakers:
participant.speaker = False
if (participant.name, participant.email) in speakers:
participant.speaker = True
if user[others['ticket_type']]:
participant.purchases.append(
user[others['ticket_type']].strip())
if 'addons' in others and others['addons'] and user[
others['addons']]:
participant.purchases = participant.purchases + (
user[others['addons']]).strip().split(',')
for i, purchase in enumerate(participant.purchases):
participant.purchases[i] = purchase.strip().replace(
u'Apr 18 - 19',
u'May 16 - 17').replace(u'Apr 16 - 17',
u'May 14 - 15').replace(
u'Apr 16 - 19',
u'May 14 - 17')
if u"T-shirt" in participant.purchases or u"Corporate" in participant.purchases:
participant.purchased_tee = True
participant.purchases = ', '.join(
list(set(participant.purchases)))
participant.online_reg = True
if participant.ticket_number:
tickets.append(participant.ticket_number)
try:
if new:
db.session.add(participant)
db.session.commit()
if new:
added[0] = added[0] + 1
ret.append("Added " +
participant.name.encode('utf-8'))
else:
updated[0] = updated[0] + 1
ret.append("Updated " +
participant.name.encode('utf-8'))
except Exception as e:
ret.append("Error adding/updating " +
participant.name.encode('utf-8') + ': ' +
str(e))
failed.append(
participant.name.encode('utf-8') + '(' +
participant.email.encode('utf-8') + '): ' + str(e))
db.session.rollback()
def indexof(name):
try:
return headers.index(name)
except ValueError:
return None
columns = dict(ticket_number=indexof(u'ticketnumber'),
name=indexof(u'name'),
email=indexof(u'email'),
company=indexof(u'company'),
job=indexof(u'jobtitle'),
city=indexof(u'city'),
phone=indexof(u'phone'),
twitter=indexof(u'twitterhandle'),
regdate=indexof(u'date'),
order_id=indexof(u'orderid'))
others = dict(ticket_type=indexof(u'ticketname'),
addons=indexof(u'addonspurchased'))
for user in users:
process_ticket(user)
ret.append("Done with Participants")
ret.append("Starting Guests")
f = StringIO.StringIO(guests_csv)
headers = [
make_name(field).replace(u'-', u'').replace(u'\n', u'')
for field in f.next().split(',')
]
users = unicodecsv.reader(f, delimiter=',')
columns = dict(ticket_number=indexof(u'ticketnumber'),
name=indexof(u'name'),
email=indexof(u'email'),
company=indexof(u'company'),
job=indexof(u'jobtitle'),
city=indexof(u'city'),
phone=indexof(u'phone'),
twitter=indexof(u'twitterhandle'),
regdate=indexof(u'confirmedon'))
others = dict(ticket_type=indexof(u'ticket'))
for user in users:
process_ticket(user)
ret.append("Done with Guests")
if eventframe_data:
ret.append("Starting Eventframe RSVP data")
columns = dict(ticket_number='ticket',
name='ticket_name',
email='ticket_email',
phone='ticket_phone',
company='ticket_company',
job='ticket_jobtitle',
city='ticket_city',
twitter='ticket_twitter')
others = dict(ticket_type='ticket_type')
for user in eventframe_data:
if user['status'] == "Y" and 'ticket' in user:
process_ticket(user)
ret.append("Done with RSVPs")
ret.append("Removing deleted tickets")
participants = Participant.query.filter(
~Participant.ticket_number.in_(tickets),
Participant.online_reg == True,
Participant.event_id == event.id).all()
for participant in participants:
try:
db.session.delete(participant)
db.session.commit()
deleted = deleted + 1
ret.append("Delete: " + participant.name.encode('utf-8'))
except Exception as e:
ret.append("Error deleting " +
participant.name.encode('utf-8') + ':' + e)
ret.append("Deleting complete")
for participant in Participant.query.filter_by(event=event,
image='LOAD').all():
ret.append("Loading gravatar image for %s" % (participant.email))
try:
response = requests.get(
"http://www.gravatar.com/avatar/" +
md5(participant.email.lower()).hexdigest(),
params={
'd': '404',
's': '400'
})
if response.status_code == 404:
participant.image = None
ret.append('Image not present')
else:
participant.image = b64encode(response.content)
ret.append('Image loaded: %s' % participant.image)
db.session.commit()
except Exception as e:
ret.append('Error: ' + str(e))
return json.dumps(
dict(added=added[0],
updated=updated[0],
deleted=deleted,
failed=failed,
trace=ret))
# Import module statements
import re, sys, httplib, urllib
from bs4 import BeautifulSoup
from mechanize import ParseResponse, urlopen, urljoin
# To read the information about a device, you will need the proper device ID. Enter device ID from the web URL in place of XXXXXXXXXX.
if len(sys.argv) == 1:
uri = "https://graph.api.smartthings.com/device/show/XXXXXXXXXX"
else:
uri = sys.argv[1]
# Open URL and parse out information to see what forms are required to login
response = urlopen(urljoin(uri, ""))
forms = ParseResponse(response, backwards_compat=False)
form = forms[0]
# Uncomment line below to print out the form names used for logging in
#print form
# Enter information for username and password. Must put information in quotes.
form["j_username"] = "******"
form["j_password"] = "******"
# Send login information and read webpage data
text_data = urlopen(form.click()).read()
# START DATA PARSING
# Get HTML code of web page to parse through
soup = BeautifulSoup(text_data)
# Find line that has desired data. This case grabs the line with temperature.
text = soup.find_all(text=re.compile("temperature"))
def getStars(age, zeta):
"""
Returns a list of stars (for use by e.g. phoenix grid) with necessary parameters to determine
their characteristics and magnitude.
Inputs:
-------
age: float
The age of the stellar population, in years
zeta: float
The metal content of the stars, where the Sun has 0.019 as its metal content.
Output:
-------
star_list: list of lists
The output stars. The list has the following columns:
Z: [M/H]
Age (Gyr)
M_ini (M_\odot)
M_act (M_\odot)
Te
log(g)
int_IMF
Johnson,I
"""
result_str = re.compile(r"The results are available at <a href=(.*?)>output\d*\.dat</a>")
request = mechanize.Request("http://stev.oapd.inaf.it/cgi-bin/cmd_2.5")
response = mechanize.urlopen(request)
forms = mechanize.ParseResponse(response,backwards_compat=False)
response.close()
form = forms[0]
#The reasoning here is that I can *get* Johnson filters in JWST pysynphot, but can't figure
#out how to do the Spitzer equivalents.
# form["photsys_file"] = ["tab_mag_odfnew/tab_mag_2mass_spitzer_wise.dat"]
form["photsys_file"] = ["tab_mag_odfnew/tab_mag_ubvrijhk.dat"]
#0 = single isochrone, single metallicity.
#1 = sequence of isochrones, different ages, constant metallicity
#2 = sequence of isochrones, constant age, different metallicities
form["isoc_val"] = ["0"]
#Age for single-single
form["isoc_age"] = '%g' % (age)
form["isoc_zeta"] = '%g' % (zeta)
request2 = form.click()
response2 = mechanize.urlopen(request2)
response_value = response2.read()
response_url = response2.geturl()
match = result_str.search(response_value)
star_list = []
if match is not None:
output_url = match.group(1)
response_result = mechanize.urlopen(mechanize.urljoin(response_url,output_url))
output_lines = response_result.read().split("\n")
output_lines = output_lines[13:]
for line in output_lines:
if line != "":
#Z, log(age/yr), M_ini, M_act, logL/Lo, logTe, logG, mbol, U, B, V, R, I, J, H, K, int_IMF, stage
items = line.split()
star = [None]
star.append(getZ(float(items[0])))
star.append(10**float(items[1]))
star.append(float(items[2]))
star.append(float(items[3]))
star.append(10**float(items[5]))
star.append(float(items[6]))
star.append(float(items[6]))
star.append(float(items[16]))
star.append(float(items[12]))
star_list.append(star)
return star_list
#!/usr/bin/env python
import sys
import mechanize
if len(sys.argv) == 1:
uri = "http://wwwsearch.sourceforge.net/"
else:
uri = sys.argv[1]
request = mechanize.Request(mechanize.urljoin(uri, "mechanize/example.html"))
response = mechanize.urlopen(request)
forms = mechanize.ParseResponse(response, backwards_compat=False)
response.close()
## f = open("example.html")
## forms = mechanize.ParseFile(f, "http://example.com/example.html",
## backwards_compat=False)
## f.close()
form = forms[0]
print(form) # very useful!
# A 'control' is a graphical HTML form widget: a text entry box, a
# dropdown 'select' list, a checkbox, etc.
# Indexing allows setting and retrieval of control values
original_text = form["comments"] # a string, NOT a Control instance
form["comments"] = "Blah."
# Controls that represent lists (checkbox, select and radio lists) are
# ListControl instances. Their values are sequences of list item names.
#!/usr/bin/env python
import sys
from mechanize import ParseResponse, urlopen, urljoin
if len(sys.argv) == 1:
uri = "http://wwwsearch.sourceforge.net/"
else:
uri = sys.argv[1]
response = urlopen(urljoin(uri, "mechanize/example.html"))
forms = ParseResponse(response, backwards_compat=False)
form = forms[0]
print(form)
form["comments"] = "Thanks, Gisle"
# form.click() returns a mechanize.Request object
# (see HTMLForm.click.__doc__ if you want to use only the forms support, and
# not the rest of mechanize)
print(urlopen(form.click()).read())
def _open(self, page="karta.php"):
# import pdb; pdb.set_trace()
self.browser.open(mechanize.urljoin(self.url, page))
self.browser._factory.is_html = True
# This script logs you in JIRA and prints the landing page. import sys from mechanize import ParseResponse, urlopen, urljoin # build the login URL and get the page uri = "https://YOUR JIRA URL" response = urlopen(urljoin(uri, "login.jsp")) # get the first form in the page and print it forms = ParseResponse(response, backwards_compat=False) form = forms[0] print form # fills the form form["os_username"] = "******" form["os_password"] = "******" # press click and print response print urlopen(form.click()).read()
from instance import config
import sys
import re
import StringIO
import unicodecsv
import dateutil.parser
from .mechanizer import Mechanizer
import getpass
from helpers import title, yes_no
from urlparse import urlparse
from lxml import html
from coaster.utils import make_name
base_uri = 'http://doattend.com/'
URI = dict(
login=urljoin(base_uri, 'accounts/sign_in'),
new_event=urljoin(base_uri, 'events/new'),
overview=urljoin(base_uri, 'events/{event_id}/overview'),
edit_event=urljoin(base_uri, 'events/{event_id}/edit'),
ticketing_info=urljoin(base_uri, 'events/{event_id}/ticketing_options/edit'),
tickets=urljoin(base_uri, 'events/{event_id}/tickets'),
new_ticket=urljoin(base_uri, 'events/{event_id}/tickets/new'),
edit_ticket=urljoin(base_uri, 'events/{event_id}/tickets/{ticket_id}/edit'),
discounts=urljoin(base_uri, 'events/{event_id}/discounts'),
new_discount=urljoin(base_uri, 'events/{event_id}/discounts/new'),
reg_form=urljoin(base_uri, 'events/{event_id}/registration_form'),
orders=urljoin(base_uri, 'events/{event_id}/orders/registration_sheet.csv'),
order=urljoin(base_uri, 'events/{event_id}/orders/{order_id}'))
class DoAttend(Mechanizer):
# This script extracts all the dependencies in all the tickets of a particular search. # Again, just replace the login, password and the url of the xml view of a search. import sys import re from mechanize import ParseResponse, urlopen, urljoin # build the login URL and get the page uri = "https://YOUR JIRA URL" response = urlopen(urljoin(uri, "login.jsp")) # get the first form in the page and print it forms = ParseResponse(response, backwards_compat=False) form = forms[0] # print form # fills the form form["os_username"] = "******" form["os_password"] = "******" # press click request2 = form.click() response2 = urlopen(request2) # INSERT YOUR XML VIEW URL OF A SEARCH response3 = urlopen(urljoin(uri, "sr/jira.issueviews:searchrequest-xml/10655/SearchRequest-10655.xml?tempMax=1000")) txt=response3.read() # This part of the regular expression prints the key ids
# This script launches any search and outputs the results in xml. # To use it, just replace login / password, and point it to the url of a search. # So for example the login in this script is the xml output for new android issues. # One can redirect the output of the script to a file for further processing. import sys from mechanize import ParseResponse, urlopen, urljoin # build the login URL and get the page uri = "https://YOUR JIRA URL" response = urlopen(urljoin(uri, "login.jsp")) # get the first form in the page and print it forms = ParseResponse(response, backwards_compat=False) form = forms[0] # fills the form form["os_username"] = "******" form["os_password"] = "******" # press click request2 = form.click()
from mechanize import ParseResponse, urlopen, urljoin, LinkNotFoundError, ItemNotFoundError
from instance import config
from .mechanizer import Mechanizer
import getpass
import simplejson as json
import sys
from helpers import title
base_uri = config.get('FUNNEL_URL', 'https://funnel.hasgeek.com/')
base_lastuser_uri = config.get('LASTUSER_URL', 'https://auth.hasgeek.com/')
URI = dict(
login=urljoin(base_uri, 'login'),
lastuser_login=urljoin(base_lastuser_uri, 'login'),
proposal_json=urljoin(base_uri, '{space}/json'))
class Funnel(Mechanizer):
def __init__(self, browser=None):
super(Funnel, self).__init__(browser)
self._login()
def _login(self):
title("LOGIN")
print "Logging into Funnel..."
self.browser.open(URI['login'])
self.browser.select_form(nr=1)
form = self.browser.form
form['username'] = config.get('FUNNEL_USERNAME', None) or raw_input('Please enter your registered Funnel username: '******'password'] = config.get('FUNNEL_PASSWORD', None) or getpass.getpass('Please enter your Funnel password: '******'lastuser_login']:
print "The credentials you provided are incorrect..."
from mechanize import ParseResponse, urlopen, urljoin uri = "http://bookblog.net" response = urlopen(urljoin(uri, "/gender/genie.php")) forms = ParseResponse(response, backwards_compat=False) form = forms[0] #print form form['text'] = 'cheese' form['genre'] = ['fiction'] print urlopen(form.click()).read()