def fetch(url): result_no = 0 browser = Browser(history = NoHistory()) browser.set_handle_robots(False) browser.addheaders = USER_AGENT page = browser.open(url) html = page.read() soup = bs(html) browser.click() print soup.prettify()
class LEAMsite: def __init__(self, site, user, passwd): self.site = site self.error = False self.b = Browser() self.b.set_handle_robots(False) try: self.b.open(site) except urllib2.URLError: self.error = True return try: # try and log in from the portlet self.b.select_form('loginform') except: # try logging in from the main login page self.b.open('/'.join((site,"login_form"))) self.b.select_form(nr=1) self.b['__ac_name'] = user self.b['__ac_password'] = passwd r = self.b.open(self.b.click()) # plone changes have rendered this inoperable # capture the response and look in the content # body tag has class with login failure def checkURL(self, url): """Tries to open a URL and return true if successful (object exists) or false if error occurs. """ try: rsp = self.b.open(url) except: return False return True def getURL(self, url, data=None, filename=None): """Simple interface for retrieving the contents of a URL and writing it to a file or returning it as stringIO. """ #sys.stderr.write('getURL %s\n' % url) rsp = self.b.open(url, data) if filename: f = file("./Inputs/"+filename, 'wb') # always write to Input folder f.write(rsp.read()) f.close() return None else: return StringIO(rsp.read()) def putFileURL(self, filename, url, fileobj=None, title=None, type='text/plain'): """Simple interface for uploading a file to a plone site. <URL> should be an existing folder on the site and <filename> should be a readable file. """ #sys.stderr.write('putFileURL %s to %s\n' % (filename,url)) if not title: title = path.basename(filename) if not fileobj: fileobj = open(filename, 'rb') self.b.open('/'.join((url,'createObject?type_name=File'))) self.b.select_form("edit_form") self.b['title'] = title self.b.add_file(fileobj, type, path.basename(filename)) # form = self.b.find_control('file_delete') # form.value = ["",] self.b.submit("form.button.save") # r = self.b.open(self.b.click()) # should check that it worked def getFile(self, url, filename=None): """ getFile -- gets a file using standard download from Plone site url: The URL is pointer to the file on the Plone site filename: optional filename where data will be written """ rsp = self.b.open(url_join(url,'at_download/file')) if filename: f = open(filename, 'wb') f.write(rsp.read()) f.close() return None else: return rsp.read() def saveFile(self, url, dir=".", at_download=False): """Reads the response from a URL and saves it to a local file based on the name provided in the Content-Disposition header. The dir field specifies to the directory where the file will be stored. If the at_download flag is True then 'at_download/file' will be appended the URL. """ if at_download: rsp = self.b.open(url_join(url,'at_download/file')) else: rsp = self.b.open(url) fname = get_filename(rsp) if fname: f = open('/'.join([dir, fname]), 'wb') f.write(rsp.read()) f.close() return fname def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'): sys.stderr.write('putImageURL %s to %s\n' % (imgfile,url)) self.b.open('/'.join((url,'createObject?type_name=Image'))) self.b.select_form("edit_form") if not title: title = path.basename(imgfile) self.b['title'] = title self.b.add_file(open(imgfile), type, path.basename(imgfile)) try: # doesn't seem necessary but it is required for file upload form = self.b.find_control('image_delete') form.value = ["",] # print "This really does need to happen.." except: # print "That delete stuff never happens..." pass self.b.submit("form.button.save") def putImageURL(self, filename, url, fileobj=None, title=None, type='image/jpg'): """Simple interface for uploading a file to a plone site. <URL> should be an existing folder on the site and <filename> should be a readable file. """ #sys.stderr.write('putFileURL %s to %s\n' % (filename,url)) if not title: title = path.basename(filename) if not fileobj: fileobj = open(filename, 'rb') self.b.open('/'.join((url,'createObject?type_name=Image'))) self.b.select_form("edit_form") self.b['title'] = title self.b.add_file(fileobj, type, path.basename(filename)) # form = self.b.find_control('file_delete') # form.value = ["",] self.b.submit("form.button.save") # r = self.b.open(self.b.click()) # should check that it worked def putDocument(self, doc, url, title): """Creates a new document and add the doc (file-like object) to it.""" self.b.open('/'.join((url,'createObject?type_name=Document'))) self.b.select_form("edit_form") self.b['title'] = title doc.seek(0) self.b['text'] = doc.read() self.b.submit("form.button.save") return self.b.geturl() def getDocument(self, url): """Returns a string with the text of the current document.""" self.b.open('/'.join((url,'edit'))) self.b.select_form("edit_form") s = self.b['text'] self.b.submit("form.button.cancel") return s def editDocument(self, doc, url, title=None): """Replaces the contents of a document""" self.b.open('/'.join((url,'edit'))) self.b.select_form("edit_form") # update the title if given if title: self.b['title'] = title # slightly dangerous approach where we seek first # to test for a file-like object. If exception is # thrown then assume doc is a string. try: doc.seek(0) self.b['text'] = doc.read() except: self.b['text'] = doc self.b.submit("form.button.save") return self.b.geturl() def createFolder(self, folder, url): """Creates a folder titled <folder> at the location <url> if it doesn't already exist. Returns the full path of new folder. """ pathurl = '/'.join((url,folder.lower().replace(' ','-'))) print pathurl try: self.b.open(pathurl) except: self.b.open('/'.join((url,'createObject?type_name=Folder'))) self.b.select_form("edit_form") self.b['title'] = folder self.b.submit("form.button.save") return self.b.geturl() def editFolder(self, url, title="", description=""): """Edit the basic fields of the Folder. Mostly useful for setting the title AFTER creating the folder with a reasonable short name. """ try: self.b.open(url) except: self.error = True return None self.b.open(url+'/edit') self.b.select_form("edit_form") if title: self.b['title'] = title if description: self.b['description'] = description self.b.submit("form.button.save") return self.b.geturl() def deleteFolder(self, url): "Deletes folder and all of its contents" sys.stderr.write('DELETING folder %s\n' % url) return # Puts SimMaps on to the site def putSimMap(self, simmap, mapfile, url, simmap_file=None, mapfile_file=None, title=None, description=None, trans=.7, details=None, zoom=8): """ putSimMap Required Input: simmap, mapfile, url simmap is a file that contains the desired GIS layer mapfile is the standard .map file that maps GIS layer to image url - is the full URL to the folder where the SimMap will be stored Optional Inputs: simmap_file, mapfile_file, title, trans, details simmap_file - file-like object of the simmap if None simmap will be openned and read. mapfile_file - file-like object of the mapfile. If None mapfile will be openned and read. title - title sting of the SimMap defaults to basename(simmap) trans - transparency level details - description of SimMap as stored in the details field """ self.b.open('/'.join((url,'createObject?type_name=SimMap'))) self.b.select_form("edit_form") if not simmap_file: simmap_file = open(simmap, 'rb') if not mapfile_file: mapfile_file = open(mapfile, 'rb') if not title: title = path.splitext(path.basename(simmap))[0] self.b['title'] = str(title) self.b.add_file(simmap_file, 'application/octet-stream', path.basename(simmap), "simImage_file") self.b.add_file(mapfile_file, 'application/octet-stream', path.basename(mapfile), "mapFile_file") if description: self.b['description'] = str(description) self.b['transparency'] = str(trans) self.b['zoom'] = str(zoom) if details: self.b['details'] = str(details) self.b.submit("form.button.save") return self.b.geturl() def getSimMap(self, url): """ return the SimMap metadata Gets the metadata associated with SimMap including title, description, location, transparency, and zoom. = """ self.b.open(url+'/edit') self.b.select_form('edit_form') d = dict( title = self.b['title'], description = self.b['description'], details = self.b['details'], location = self.b['location'], transparency = self.b['Transparency'], zoom = self.b['zoom'], ) self.b.submit("form.button.cancel") return d def updateSimMap(self, url, **kwargs): """ update the SimMap metadata Keywords must match the field names from the edit form exactly, extra keywords (or mispelled ones) will be ignored. """ self.b.open(url+'/edit') self.b.select_form('edit_form') for k in kwargs: if k in self.b: self.b[k] = str(kwargs[k]) self.b.submit("form.button.save") def getSimMapData(self, url, filename=None): """ getSimMapData -- gets the data component of the the SimMap url: The URL is pointer to the SimMap on the Plone site filename: optional filename where data will be written """ bufsize = 15 * 1024 * 1024 rsp = self.b.open(url_join(url,'at_download/simImage')) if filename: f = file(filename, 'wb') while 1: b = rsp.read(bufsize) f.write(b) if len(b) < bufsize: break f.close() return None else: return StringIO(rsp.read()) def getSimMapMapfile(self, url, filename=None): """ getSimMapData -- gets the mapfile component of the the SimMap url: The URL is pointer to the SimMap on the Plone site filename: optional filename where data will be written """ rsp = self.b.open(url_join(url,'at_download/mapFile')) if filename: f = file(filename, 'wb') f.write(rsp.read()) f.close() return None else: data = StringIO() data.write(rsp.read()) return data def putProbmapCache(self, url, filename): """store precomputed probmaps as part of the Driver Set This is a temporary method until the REST interface is ready. It's really depricated before it written! """ self.b.open(url+'/edit') self.b.select_form('edit_form') with open(filename, 'rb') as f: self.b.add_file(f, 'application/octet-stream', path.basename(filename), name='probfile_file') self.b.submit("form.button.save") # def putAttrmapCache(self, url, filepathbasename, filename): # """store precomputed maps as part of the Driver Set # This is a temporary method until the REST interface # is ready. It's really depricated before it written! # """ # self.b.open(url+'/edit') # self.b.select_form('edit_form') # with open(filename, 'w') as f: # self.b.add_file(f, 'application/octet-stream', # path.basename(filepathbasename), name=filename) # self.b.submit("form.button.save") # DELETE FUNCTIONS ----------------------------------------- # These functions delete items from the LEAM Plone sites # ---------------------------------------------------------- def deleteItem(self,fname,url): """ deleteItem Deletes <fname> from the folder <url> """ print '/'.join((url,fname,'delete_confirmation')) print self.b.open('/'.join((url,fname,'delete_confirmation'))) self.b.select_form(None,None,1) print self.b.submit()
def trilegal(outputFileName, longitude = 0, latitude = 0, coordinateType = "galactic", fieldArea = 1, passband = 4, magnitudeLimit = 26, magnitudeResolution = 0.1, IMFtype = 3, includeBinaries = True, binaryFraction = 0.3, lowerBinaryMassRatio = 0.7, upperBinaryMassRatio = 1.0, extinctionType = 2, extinctionValue = 0.0378, extinctionSigma = 0.0, useThinDisc = False, useThickDisc = False, useBulge = True): """ Query the web interface of the TRILEGAL population synthesis code. The TRILEGAL webform is automatically filled and submitted. The computations are done locally on Girardi's computer. As soon as they are finished, the script retrieves the data file. Example: >>> trilegal("output.txt", longitude=3, latitude=14, coordinateType="galactic", fieldArea=1, magnitudeLimit=7, useThinDisc=True) @param outputFileName: name of file wherein trilegal output will be saved @type outputFileName: string @param longitude: galactic longitude (degrees) or right ascension (hours) @type longitude: integer @param latitude: galactic latitude (degrees) or declination (degrees) @type latitude: integer @param coordinateType: either "galactic", or "equatorial" @type coordinateType: string @param fieldArea: total field area in square degrees (max. 10 deg^2) @type fieldArea: float @param passband: U,B,V,R,I,J,H,K = 1,2,3,4,5,6,7,8 for magnitude limit @type passband: integer @param magnitudeLimit: magnitude limit in specified passband @type magnitudeLimit: float @param magnitudeResolution: Distance modulus resolution of Galaxy components (mag) @type magnitudeResolution: float @param IMFtype: type of Initial Mass Function of single stars 1 = Salpeter with cutoff at 0.01, Msun, 2 = Chabrier exponential, 3 = Chabrier lognormal, 4 = Kroupa corrected for binaries, 5 = Kroupa not corrected for binaries @type IMFtype: integer @param includeBinaries: include binaries in the population (True or False) @type includeBinaries: boolean @param binaryFraction: fraction of binaries @type binaryFraction: float @param lowerBinaryMassRatio: lower limit of binary mass fraction @type lowerBinaryMassRatio: float @param upperBinaryMassRatio: upper limit of binary mass fraction @type upperBinaryMassRatio: float @param extinctionType: Type of extinction 0: no dust extinction 1: local calibration 2: calibration at infinity @type extinctionType: integer @param extinctionValue: for a local calibration this is dAv/dr in mag/pc for the calibration at infinity this is Av at infinity in mag. @type extinctionValue: float @param extinctionSigma: 1-sigma extinction dispersion / total extinction (max. 0.3) @type extinctionSigma: float @param useThinDisk: if True use squared hyperbolic secant along z, if False don't include @type useThinDisk: boolean @param useThickDisk: if True use squared hyperbolic secant along z, if False don't include @type useThickDisk: boolean @param useBulge: if True use triaxal bulge, if False don't include @type useBulge: boolean @return None. A file is retrieved """ # The latest Trilegal web version trilegalURL = "http://stev.oapd.inaf.it/cgi-bin/trilegal" # Get the web form timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime()) print("{0}: Opening TRILEGAL web interface".format(timestamp)) myBrowser = Browser() try: myBrowser.open(trilegalURL) except: timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime()) print("{0}: Unable to open the TRILEGAL website".format(timestamp)) return myBrowser.select_form(nr=0) # there is only one form... # Fill in the form. To know how the different fields in the form are # named, we used # >>> request = mechanize.Request(trilegalURL) # >>> response = mechanize.urlopen(request) # >>> forms = mechanize.ParseResponse(response, backwards_compat=False) # >>> print forms[0] timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime()) print("{0}: Filling TRILEGAL web form".format(timestamp)) if coordinateType == "galactic": myBrowser["gal_coord"] = ["1"] myBrowser["gc_l"] = str(longitude) myBrowser["gc_b"] = str(latitude) else: myBrowser["gal_coord"] = ["2"] myBrowser["eq_alpha"] = str(longitude) myBrowser["eq_delta"] = str(latitude) myBrowser["field"] = str(fieldArea) myBrowser["icm_lim"] = str(passband) myBrowser["mag_lim"] = str(magnitudeLimit) myBrowser["mag_res"] = str(magnitudeResolution) myBrowser["binary_kind"] = [str(int(includeBinaries))] myBrowser["binary_frac"] = str(binaryFraction) myBrowser["binary_mrinf"] = str(lowerBinaryMassRatio) myBrowser["binary_mrsup"] = str(upperBinaryMassRatio) myBrowser["extinction_kind"] = [str(extinctionType)] if extinctionType == 1: myBrowser["extinction_rho_sun"] = str(extinctionValue) if extinctionType == 2: myBrowser["extinction_infty"] = str(extinctionValue) myBrowser["extinction_sigma"] = str(extinctionSigma) if useThinDisc: myBrowser["thindisk_kind"] = ["3"] else: myBrowser["thindisk_kind"] = ["0"] if useThickDisc: myBrowser["thickdisk_kind"] = ["3"] else: myBrowser["thickdisk_kind"] = ["0"] if useBulge: myBrowser["bulge_kind"] = ["2"] else: myBrowser["bulge_kind"] = ["0"] # Submit the completed form timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime()) print("{0}: Submitting completed TRILEGAL web form".format(timestamp)) nextWebPage = myBrowser.submit() # Trilegal is now computing the result. Click on the special "Refresh" # button until the webpage says that the computations are finished. timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime()) print ("{0}: Waiting until TRILEGAL computations are finished".format(timestamp)) myBrowser.select_form(nr=0) # one form on the "be patient" web page message = "Your job was finished" while (message not in nextWebPage.read()): nextWebPage = urlopen(myBrowser.click()) # click on the Refresh button myBrowser.select_form(nr=0) # select form again, so that we can make a click again sleep(5) # to not overload the website with refresh requests # Get the url of the outputfile, and retrieve it. This can take a while. timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime()) print("{0}: Retrieving TRILEGAL output file".format(timestamp)) outputLink = myBrowser.links(url_regex="lgirardi/tmp/output").next() urlretrieve(outputLink.absolute_url, outputFileName) myBrowser.close() # Save the parameters in an info file parameterInfo = """ coordinateType {0} longitude {1} latitude {2} fieldArea {3} passband {4} magnitudeLimit {5} magnitudeResolution {6} IMFtype {7} includeBinaries {8} binaryFraction {9} lowerBinaryMassRatio {10} upperBinaryMassRatio {11} extinctionType {12} extinctionValue {13} extinctionSigma {14} """.format(coordinateType, longitude, latitude, fieldArea, passband, magnitudeLimit, magnitudeResolution, IMFtype, includeBinaries, binaryFraction, lowerBinaryMassRatio, upperBinaryMassRatio, extinctionType, extinctionValue, extinctionSigma) infoFileName = "info_" + outputFileName with open(infoFileName, 'w') as infoFile: infoFile.write(parameterInfo)
class LEAMsite: def __init__(self, site, user, passwd): self.site = site self.error = False self.b = Browser() self.b.set_handle_robots(False) try: self.b.open(site) except urllib2.URLError: self.error = True return try: # try and log in from the portlet self.b.select_form('loginform') except: # try logging in from the main login page self.b.open('/'.join((site, "login_form"))) self.b.select_form(nr=1) if not user or not passwd: raise ValueError('user and password are required') self.b['__ac_name'] = user self.b['__ac_password'] = passwd r = self.b.open(self.b.click()) # plone changes have rendered this inoperable # capture the response and look in the content # #if 'logged_in' in path.split(self.b.geturl()): # sys.stderr.write("Error: unable to login to LEAM Plone site\n") # sys.exit(1) def getURL(self, url, data=None, filename=None): """Simple interface for retrieving the contents of a URL and writing it to a file or returning it as stringIO. """ #sys.stderr.write('getURL %s\n' % url) rsp = self.b.open(url, data) if filename: f = file(filename, 'wb') f.write(rsp.read()) f.close() return None else: return StringIO(rsp.read()) def putFileURL(self, filename, url, fileobj=None, title=None, type='text/plain'): """Simple interface for uploading a file to a plone site. <URL> should be an existing folder on the site and <filename> should be a readable file. """ #sys.stderr.write('putFileURL %s to %s\n' % (filename,url)) if not title: title = path.basename(filename) if not fileobj: fileobj = open(filename, 'rb') self.b.open('/'.join((url, 'createObject?type_name=File'))) self.b.select_form("edit_form") self.b['title'] = title self.b.add_file(fileobj, type, path.basename(filename)) # form = self.b.find_control('file_delete') # form.value = ["",] self.b.submit("form.button.save") # r = self.b.open(self.b.click()) # should check that it worked def getFile(self, url, filename=None): """ getFile -- gets a file using standard download from Plone site url: The URL is pointer to the file on the Plone site filename: optional filename where data will be written """ rsp = self.b.open(url_join(url, 'at_download/file')) if filename: f = open(filename, 'wb') f.write(rsp.read()) f.close() return None else: return rsp.read() def saveFile(self, url, dir=".", at_download=False): """Reads the response from a URL and saves it to a local file based on the name provided in the Content-Disposition header. The dir field specifies to the directory where the file will be stored. If the at_download flag is True then 'at_download/file' will be appended the URL. """ if at_download: rsp = self.b.open(url_join(url, 'at_download/file')) else: rsp = self.b.open(url) fname = get_filename(rsp) if fname: f = open('/'.join([dir, fname]), 'wb') f.write(rsp.read()) f.close() return fname def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'): sys.stderr.write('putImageURL %s to %s\n' % (imgfile, url)) self.b.open('/'.join((url, 'createObject?type_name=Image'))) self.b.select_form("edit_form") if not title: title = path.basename(imgfile) self.b['title'] = title self.b.add_file(open(imgfile), type, path.basename(imgfile)) try: # doesn't seem necessary but it is required for file upload form = self.b.find_control('image_delete') form.value = [ "", ] # print "This really does need to happen.." except: # print "That delete stuff never happens..." pass self.b.submit("form.button.save") def putImageURL(self, filename, url, fileobj=None, title=None, type='image/jpg'): """Simple interface for uploading a file to a plone site. <URL> should be an existing folder on the site and <filename> should be a readable file. """ #sys.stderr.write('putFileURL %s to %s\n' % (filename,url)) if not title: title = path.basename(filename) if not fileobj: fileobj = open(filename, 'rb') self.b.open('/'.join((url, 'createObject?type_name=Image'))) self.b.select_form("edit_form") self.b['title'] = title self.b.add_file(fileobj, type, path.basename(filename)) # form = self.b.find_control('file_delete') # form.value = ["",] self.b.submit("form.button.save") # r = self.b.open(self.b.click()) # should check that it worked def putDocument(self, doc, url, title): """Creates a new document and add the doc (file-like object) to it.""" self.b.open('/'.join((url, 'createObject?type_name=Document'))) self.b.select_form("edit_form") self.b['title'] = title doc.seek(0) self.b['text'] = doc.read() self.b.submit("form.button.save") return self.b.geturl() def getDocument(self, url): """Returns a string with the text of the current document.""" self.b.open('/'.join((url, 'edit'))) self.b.select_form("edit_form") s = self.b['text'] self.b.submit("form.button.cancel") return s def editDocument(self, doc, url, title=None): """Replaces the contents of a document""" self.b.open('/'.join((url, 'edit'))) self.b.select_form("edit_form") # update the title if given if title: self.b['title'] = title # slightly dangerous approach where we seek first # to test for a file-like object. If exception is # thrown then assume doc is a string. try: doc.seek(0) self.b['text'] = doc.read() except: self.b['text'] = doc self.b.submit("form.button.save") return self.b.geturl() def createFolder(self, folder, url): """Creates a folder titled <folder> at the location <url> if it doesn't already exist. Returns the full path of new folder. """ pathurl = '/'.join((url, folder.lower().replace(' ', '-'))) try: self.b.open(pathurl) except: self.b.open('/'.join((url, 'createObject?type_name=Folder'))) self.b.select_form("edit_form") self.b['title'] = folder self.b.submit("form.button.save") return self.b.geturl() def editFolder(self, url, title="", description=""): """Edit the basic fields of the Folder. Mostly useful for setting the title AFTER creating the folder with a reasonable short name. """ try: self.b.open(url) except: self.error = True return None self.b.open(url + '/edit') self.b.select_form("edit_form") if title: self.b['title'] = title if description: self.b['description'] = description self.b.submit("form.button.save") return self.b.geturl() def deleteFolder(self, url): "Deletes folder and all of its contents" sys.stderr.write('DELETING folder %s\n' % url) return # Puts SimMaps on to the site def putSimMap(self, simmap, mapfile, url, simmap_file=None, mapfile_file=None, title=None, description=None, trans=.7, details=None, zoom=11): """ putSimMap Required Input: simmap, mapfile, url simmap is a file that contains the desired GIS layer mapfile is the standard .map file that maps GIS layer to image url - is the full URL to the folder where the SimMap will be stored Optional Inputs: simmap_file, mapfile_file, title, trans, details simmap_file - file-like object of the simmap if None simmap will be openned and read. mapfile_file - file-like object of the mapfile. If None mapfile will be openned and read. title - title sting of the SimMap defaults to basename(simmap) trans - transparency level details - description of SimMap as stored in the details field """ self.b.open('/'.join((url, 'createObject?type_name=SimMap'))) self.b.select_form("edit_form") if not simmap_file: simmap_file = open(simmap, 'rb') if not mapfile_file: mapfile_file = open(mapfile, 'rb') if not title: title = path.splitext(path.basename(simmap))[0] self.b['title'] = str(title) self.b.add_file(simmap_file, 'application/octet-stream', path.basename(simmap), "simImage_file") self.b.add_file(mapfile_file, 'application/octet-stream', path.basename(mapfile), "mapFile_file") if description: self.b['description'] = str(description) self.b['transparency'] = str(trans) self.b['zoom'] = str(zoom) if details: self.b['details'] = str(details) self.b.submit("form.button.save") return self.b.geturl() def getSimMap(self, url): """ getSimMap Gets the metadata associated with SimMap including title, description, location, transparency, and zoom. """ self.b.open(url + '/edit') self.b.select_form('edit_form') d = dict( title=self.b['title'], description=self.b['description'], details=self.b['details'], location=self.b['location'], transparency=self.b['Transparency'], zoom=self.b['zoom'], ) self.b.submit("form.button.cancel") return d def getSimMapData(self, url, filename=None): """ getSimMapData -- gets the data component of the the SimMap url: The URL is pointer to the SimMap on the Plone site filename: optional filename where data will be written """ bufsize = 15 * 1024 * 1024 rsp = self.b.open(url_join(url, 'get_layer')) if filename: f = file(filename, 'wb') while 1: b = rsp.read(bufsize) f.write(b) if len(b) < bufsize: break f.close() return None else: return StringIO(rsp.read()) def getSimMapMapfile(self, url, filename=None): """ getSimMapData -- gets the mapfile component of the the SimMap url: The URL is pointer to the SimMap on the Plone site filename: optional filename where data will be written """ rsp = self.b.open(url_join(url, 'get_mapfile')) if filename: f = file(filename, 'wb') f.write(rsp.read()) f.close() return None else: data = StringIO() data.write(rsp.read()) return data # DELETE FUNCTIONS ----------------------------------------- # These functions delete items from the LEAM Plone sites # ---------------------------------------------------------- def deleteItem(self, fname, url): """ deleteItem Deletes <fname> from the folder <url> """ print '/'.join((url, fname, 'delete_confirmation')) print self.b.open('/'.join((url, fname, 'delete_confirmation'))) self.b.select_form(None, None, 1) print self.b.submit()
from bs4 import BeautifulSoup from mechanize import Browser import re import mechanize url = "http://127.0.0.1/webgoat/attack" br = Browser() br.set_handle_robots(False) br.add_password(url, "guest", "guest") br.open(url) br.select_form(nr=0) sub = br.click(type="submit", nr=0) br.open(sub) soup = BeautifulSoup(br.response().read(), 'lxml') link = soup.a all_links = link.find_all_next('a') print "Welche SQL-Operation?\n" i = 0 SQL_links = [] for link in all_links: if re.search(r'menu=1200', str(link)): i = i + 1 SQL_links.append(str(link['href']))
def attack(address, htmlObject): br = Browser() br.open(address) br.click(htmlObject)