def newBrowser(self):
# Create new browsers all the time because its data structures grow
# unboundedly (texas#135)
br = Browser()
br.add_password(self.hostname, self.username, self.password)
br.set_handle_robots(None)
return br
def newBrowser(self): # Create new browsers all the time because its data structures grow # unboundedly (texas#135) br = Browser() br.add_password(self.hostname, self.username, self.password) br.set_handle_robots(None) return br
def GetServerConfigFile(url_aastra, return_file):
br = Browser()
br.add_password(url_aastra, "admin", "22222")
try:
br.retrieve(url_aastra + "/servercfg.html", return_file)
return True
except:
log.warn("Maybe isn't a aastra phone? Are you Sure?")
return False
def GetServerConfigFile(url_aastra, return_file):
br = Browser()
br.add_password(url_aastra, "admin", "22222")
try:
br.retrieve(url_aastra + "/servercfg.html", return_file)
return True
except:
log.warn("Maybe isn't a aastra phone? Are you Sure?")
return False
def RemoveLocalConfig(url_aastra):
br = Browser()
br.add_password(url_aastra, "admin", "22222")
br.open(url_aastra + "/reset.html")
br.select_form(nr=0)
resetOption = br.form.controls[3]
resetOption.readonly = False
br.form["resetOption"] = "2"
response = br.submit()
print response.read()
def FactoryReset(url_aastra):
br = Browser()
br.add_password(url_aastra, "admin", "22222")
br.open(url_aastra + "/reset.html")
br.select_form(nr=0)
c = br.form.controls[3]
c.readonly = False
br.form["resetOption"] = "1"
response = br.submit()
print response.read()
def RemoveLocalConfig(url_aastra):
br = Browser()
br.add_password(url_aastra, "admin", "22222")
br.open(url_aastra + "/reset.html")
br.select_form(nr=0)
resetOption=br.form.controls[3]
resetOption.readonly=False
br.form["resetOption"]="2"
response=br.submit()
print response.read()
def FactoryReset(url_aastra):
br = Browser()
br.add_password(url_aastra, "admin", "22222")
br.open(url_aastra + "/reset.html")
br.select_form(nr=0)
c=br.form.controls[3]
c.readonly=False
br.form["resetOption"]="1"
response=br.submit()
print response.read()
def create_browsing_context():
if not os.path.exists('corpas.focloir.ie.credentials'):
# untried
username = input('corpas.focloir.ie username:'******'corpas.focloir.ie password:'******'corpas.focloir.ie.credentials', 'rw') as f:
f.write(username + '\n' + password)
with open('corpas.focloir.ie.credentials', 'r') as f:
username, password = f.read().split('\n')
br = Browser()
br.set_handle_robots(False)
#br.open('http://corpas.focloir.ie/')
br.open('http://focloir.sketchengine.co.uk/')
br.add_password(
'http://focloir.sketchengine.co.uk/auth/run.cgi/simple_search?home=1',
username, password)
for link in br.links():
if link.text.lower().replace(' ', '') == 'login':
br.follow_link(link)
return br
def get_times(dublinked_username,
dublinked_password,
operator_id,
realtime_id,
realtime_results=None):
predicted_times = defaultdict(list)
br = Browser()
br.set_handle_robots(False)
br.addheaders = [
("User-agent", 'Python script, run by %s' % (dublinked_username)),
("Referer", 'https://github.com/Dublin-Public-Transport-Developers')
]
url = 'http://www.dublinked.ie/cgi-bin/rtpi/realtimebusinformation?stopid=%s&format=json' % (
realtime_id)
br.add_password(url, dublinked_username, dublinked_password)
stime = time()
br.open(url)
request_time_ms = int(1000 * (time() - stime))
log.debug("### Downloaded dublinked %s, stop id %s in %dms" %
(operator_id, realtime_id, request_time_ms))
json_contents = json.loads(br.response().read())
predicted_times = defaultdict(list)
if json_contents['errorcode'] != '0':
if json_contents['errorcode'] != '1':
error_warning = "dublinked error %s [%s]: (%s)" % (
json_contents['errorcode'], realtime_id,
json_contents.get('errormessage', ''))
log.warning(error_warning)
else:
for result_row in json_contents['results']:
warn = ''
route_desc = clean_route_desc(result_row['route']).lower()
real_t = to_timedelta(result_row['departuredatetime'])
sched_t = to_timedelta(result_row['scheduleddeparturedatetime'])
predicted_times[route_desc].append((sched_t, real_t, bool(warn)))
if realtime_results:
realtime_results.put(predicted_times)
log.debug("### Put results %s %s" % (realtime_id, time() - stime))
return predicted_times
def setupBrowser(br: Browser, url: str, httpUsername: str,
httpPassword: str):
if httpUsername is not None and httpPassword is not None:
br.set_handle_robots(False)
br.add_password(url, httpUsername, httpPassword)
from bs4 import BeautifulSoup
from mechanize import Browser
import re
import mechanize
url = "http://127.0.0.1/webgoat/attack"
br = Browser()
br.set_handle_robots(False)
br.add_password(url, "guest", "guest")
br.open(url)
br.select_form(nr=0)
sub = br.click(type="submit", nr=0)
br.open(sub)
soup = BeautifulSoup(br.response().read(), 'lxml')
link = soup.a
all_links = link.find_all_next('a')
print "Welche SQL-Operation?\n"
i = 0
SQL_links = []
for link in all_links:
if re.search(r'menu=1200', str(link)):
i = i + 1
SQL_links.append(str(link['href']))
def initializer(ip, password):
url = "http://" + ip + "/"
username = '******'
browser = Browser()
browser.add_password(url, username, password)
browser.open(url)
class OutlookWebScraper():
def __init__(self, domain, username, password):
logger.debug(locals())
self.domain = domain
self.username, self.password = username, password
self.is_logged_in = False
self.base_href = None
self.browser = Browser() # home to the mechanize headless browser-like entity
self.folder_cache = {} # a dict for storing folder names and lists of messages
self.message_cache = {} # a dict for storing message IDs and content
def add_to_cache(self, folder_name=None, message_urls=None, msgid=None, payload=None):
if folder_name: # simple - stores the message IDs from a folder with the folder name as a key
self.folder_cache[folder_name] = message_urls
return self.folder_cache[folder_name]
if msgid: # even simpler - store the raw e-mail under the message id
self.message_cache[msgid] = payload
return self.message_cache[msgid]
def find_in_cache(self, folder_name=None, msgid=None):
if msgid: # quick key-in-d lookup
if msgid in self.message_cache:
return self.message_cache[msgid]
if folder_name:
if folder_name in self.folder_cache:
return self.folder_cache[folder_name]
else: # i.e. no message of that id or no folder of that name
return False
def remove_from_cache(self, folder_name=None, msgid=None):
if msgid:
cache = find_in_cache(msgid=msgid) # search for the message
if cache:
del self.message_cache[msgid] # remove it
if not folder_name: # if that's it..
return True
# if a folder name was provided, or no message found, though..
if folder_name:
cache = find_in_cache(folder_name=folder_name) # search for a folder
if cache:
del self.folder_cache[folder_name] # if found, kill it
for id in cache:
del self.message_cache[id] # and remove the messages
return True
else:
return False
else: # nothing was found
return False
def flush_cache(self):
self.folder_cache = {} # really simple
self.message_cache = {}
return True
def login(self):
logger.debug(locals())
destination = urlparse.urljoin(self.domain, 'exchange/')
self.browser.add_password(destination, self.username, self.password)
self.browser.open(destination) # it should just work with basic auth, but let's deal with form as well
self.browser.select_form('logonForm')
self.browser['username'] = self.username
self.browser['password'] = self.password
self.browser.submit()
if 'You could not be logged on to Outlook Web Access' in self.browser.response().read():
raise InvalidLogin
m = self.browser.links().next()
if not m.base_url:
raise RetrievalError, "Couldn't find <base href> on page after logging in."
self.base_href = m.base_url
self.is_logged_in = True
def inbox(self, refresh=None):
"""
Returns the message IDs for all messages in the
Inbox, regardless of whether they've already been read. setting refresh forces an update.
"""
logger.debug(locals())
if refresh: # refresh kwarg. if set, forces a fresh load of the folder
return self.get_folder('Inbox', refresh=True)
else:
return self.get_folder('Inbox')
def get_folder(self, folder_name, refresh=None):
"""
Returns the message IDs for all messages in the
folder with the given name, regardless of whether the messages have
already been read. The folder name is case insensitive. setting refresh forces an update. if not set, cached messages
will be returned if they exist.
"""
logger.debug(locals())
if not refresh: # look in the cache
message_urls = find_in_cache(folder_name=folder_name)
if message_urls:
return message_urls
# if not found or not used, proceed
if not self.is_logged_in:
self.login()
url = self.base_href + urllib.quote(folder_name) + '/?Cmd=contents'
self.browser.open(url)
message_urls = [link.url for link in self.browser.links() if '.EML' in link.url]
if ' of 1' in self.browser.response().read(): # test for multiple pages
add_to_cache(folder_name=folder_name, message_urls=message_urls) # cache it
return message_urls
else:
last_message_urls = message_urls # if you ask for an out-of-range page, you get the last page
flag = True
while flag: # page through the folder, testing to see if we've got the last one yet
next_url = [link.url for link in self.browser.links() if 'Next Page' in link.text][0]
self.browser.open(next_url) #find and use next page url
murls = [(link.url).replace((self.base_href), '') for link in self.browser.links() if '.EML' in link.url] #extract data
if murls == last_message_urls:
flag = False # check to see if identical with the last one
last_message_urls = murls
message_urls.extend(murls) # build the list
add_to_cache(folder_name=folder_name, message_urls=message_urls)
return message_urls # if you get the same page you've finished
def get_message(self, msgid):
"Returns the raw e-mail for the given message ID."
logger.debug(locals())
payload = find_in_cache(msgid=msgid) #check the cache
if payload:
return payload
if not self.is_logged_in:
self.login()
# Sending the "Translate=f" HTTP header tells Outlook to include
# full e-mail headers. Figuring that out took way too long.
self.browser.addheaders = [('Translate', 'f')]
payload = self.browser.open(self.base_href + msgid + '?Cmd=body')
add_to_cache(msgid=msgid, payload=payload) #cache the results
return payload
def delete_message(self, msgid):
"Deletes the e-mail with the given message ID."
logger.debug(locals())
if not self.is_logged_in:
self.login()
msgid = msgid.replace('?Cmd=open', '')
delete = self.browser.open(self.base_href + msgid, urllib.urlencode({
'MsgId': msgid,
'Cmd': 'delete',
'ReadForm': '1',
}))
remove_from_cache(msgid=msgid) #and remove from cache
return delete
class OutlookWebScraper():
def __init__(self, domain, username, password):
logger.debug(locals())
self.domain = domain
self.username, self.password = username, password
self.is_logged_in = False
self.base_href = None
self.browser = Browser(
) # home to the mechanize headless browser-like entity
self.folder_cache = {
} # a dict for storing folder names and lists of messages
self.message_cache = {} # a dict for storing message IDs and content
def add_to_cache(self,
folder_name=None,
message_urls=None,
msgid=None,
payload=None):
if folder_name: # simple - stores the message IDs from a folder with the folder name as a key
self.folder_cache[folder_name] = message_urls
return self.folder_cache[folder_name]
if msgid: # even simpler - store the raw e-mail under the message id
self.message_cache[msgid] = payload
return self.message_cache[msgid]
def find_in_cache(self, folder_name=None, msgid=None):
if msgid: # quick key-in-d lookup
if msgid in self.message_cache:
return self.message_cache[msgid]
if folder_name:
if folder_name in self.folder_cache:
return self.folder_cache[folder_name]
else: # i.e. no message of that id or no folder of that name
return False
def remove_from_cache(self, folder_name=None, msgid=None):
if msgid:
cache = find_in_cache(msgid=msgid) # search for the message
if cache:
del self.message_cache[msgid] # remove it
if not folder_name: # if that's it..
return True
# if a folder name was provided, or no message found, though..
if folder_name:
cache = find_in_cache(
folder_name=folder_name) # search for a folder
if cache:
del self.folder_cache[folder_name] # if found, kill it
for id in cache:
del self.message_cache[id] # and remove the messages
return True
else:
return False
else: # nothing was found
return False
def flush_cache(self):
self.folder_cache = {} # really simple
self.message_cache = {}
return True
def login(self):
logger.debug(locals())
destination = urlparse.urljoin(self.domain, 'exchange/')
self.browser.add_password(destination, self.username, self.password)
self.browser.open(
destination
) # it should just work with basic auth, but let's deal with form as well
self.browser.select_form('logonForm')
self.browser['username'] = self.username
self.browser['password'] = self.password
self.browser.submit()
if 'You could not be logged on to Outlook Web Access' in self.browser.response(
).read():
raise InvalidLogin
m = self.browser.links().next()
if not m.base_url:
raise RetrievalError, "Couldn't find <base href> on page after logging in."
self.base_href = m.base_url
self.is_logged_in = True
def inbox(self, refresh=None):
"""
Returns the message IDs for all messages in the
Inbox, regardless of whether they've already been read. setting refresh forces an update.
"""
logger.debug(locals())
if refresh: # refresh kwarg. if set, forces a fresh load of the folder
return self.get_folder('Inbox', refresh=True)
else:
return self.get_folder('Inbox')
def get_folder(self, folder_name, refresh=None):
"""
Returns the message IDs for all messages in the
folder with the given name, regardless of whether the messages have
already been read. The folder name is case insensitive. setting refresh forces an update. if not set, cached messages
will be returned if they exist.
"""
logger.debug(locals())
if not refresh: # look in the cache
message_urls = find_in_cache(folder_name=folder_name)
if message_urls:
return message_urls
# if not found or not used, proceed
if not self.is_logged_in:
self.login()
url = self.base_href + urllib.quote(folder_name) + '/?Cmd=contents'
self.browser.open(url)
message_urls = [
link.url for link in self.browser.links() if '.EML' in link.url
]
if ' of 1' in self.browser.response().read(
): # test for multiple pages
add_to_cache(folder_name=folder_name,
message_urls=message_urls) # cache it
return message_urls
else:
last_message_urls = message_urls # if you ask for an out-of-range page, you get the last page
flag = True
while flag: # page through the folder, testing to see if we've got the last one yet
next_url = [
link.url for link in self.browser.links()
if 'Next Page' in link.text
][0]
self.browser.open(next_url) #find and use next page url
murls = [(link.url).replace((self.base_href), '')
for link in self.browser.links()
if '.EML' in link.url] #extract data
if murls == last_message_urls:
flag = False # check to see if identical with the last one
last_message_urls = murls
message_urls.extend(murls) # build the list
add_to_cache(folder_name=folder_name, message_urls=message_urls)
return message_urls # if you get the same page you've finished
def get_message(self, msgid):
"Returns the raw e-mail for the given message ID."
logger.debug(locals())
payload = find_in_cache(msgid=msgid) #check the cache
if payload:
return payload
if not self.is_logged_in:
self.login()
# Sending the "Translate=f" HTTP header tells Outlook to include
# full e-mail headers. Figuring that out took way too long.
self.browser.addheaders = [('Translate', 'f')]
payload = self.browser.open(self.base_href + msgid + '?Cmd=body')
add_to_cache(msgid=msgid, payload=payload) #cache the results
return payload
def delete_message(self, msgid):
"Deletes the e-mail with the given message ID."
logger.debug(locals())
if not self.is_logged_in:
self.login()
msgid = msgid.replace('?Cmd=open', '')
delete = self.browser.open(
self.base_href + msgid,
urllib.urlencode({
'MsgId': msgid,
'Cmd': 'delete',
'ReadForm': '1',
}))
remove_from_cache(msgid=msgid) #and remove from cache
return delete
Username = username
Password = password
import re
import sys
print "Please enter a description for this bug:"
desc = sys.stdin.read()
print "Thank you; your bug is now being submitted."
from mechanize import Browser
br = Browser()
br.add_password("http://trac.newartisans.com/ledger/login",
Username, Password)
print "Logging in to the Trac ..."
br.open("http://trac.newartisans.com/ledger/login")
assert br.viewing_html()
# follow second link with element text matching regular expression
print "Opening the New Ticket page ..."
resp1 = br.open("http://trac.newartisans.com/ledger/newticket")
newticket = None
index = 0
for form in br.forms():
if index == 1:
newticket = form
break
def add_password(self, *args, **kwargs):
B.add_password(self, *args, **kwargs)
self._clone_actions['add_password'] = ('add_password', args, kwargs)
def add_password(self, *args, **kwargs):
B.add_password(self, *args, **kwargs)
self._clone_actions['add_password'] = ('add_password', args, kwargs)
import urllib
import time
from bs4 import *
import re
import mechanize
from mechanize import Browser
from socket import error as SocketError
mech = Browser()
mech.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
mech.add_password('https://www.nebraska.gov/', 'USERNAME', 'PASSWORD')
mech.set_handle_robots(False)
baseurl = "https://www.nebraska.gov/justice/name.cgi"
z = open('lawyer_name.txt', 'wb')
firstcounter = 0
while firstcounter < 5:
try:
mech.open(baseurl)
firstcounter = 5
print "Got nebraska.gov"
except mechanize.HTTPError, e:
print "Broke on first loop. I'll try to load the search form again."
firstcounter += 1
time.sleep(30)
print "Taking a 30-second nap."
except SocketError, e:
print "Broke on first loop w/ socket error. I'll try to load the search form again."
