def fetch(url):
result_no = 0
browser = Browser(history = NoHistory())
browser.set_handle_robots(False)
browser.addheaders = USER_AGENT
page = browser.open(url)
html = page.read()
soup = bs(html)
browser.click()
print soup.prettify()
class LEAMsite:
def __init__(self, site, user, passwd):
self.site = site
self.error = False
self.b = Browser()
self.b.set_handle_robots(False)
try:
self.b.open(site)
except urllib2.URLError:
self.error = True
return
try:
# try and log in from the portlet
self.b.select_form('loginform')
except:
# try logging in from the main login page
self.b.open('/'.join((site,"login_form")))
self.b.select_form(nr=1)
self.b['__ac_name'] = user
self.b['__ac_password'] = passwd
r = self.b.open(self.b.click())
# plone changes have rendered this inoperable
# capture the response and look in the content
# body tag has class with login failure
def checkURL(self, url):
"""Tries to open a URL and return true if successful (object exists)
or false if error occurs.
"""
try:
rsp = self.b.open(url)
except:
return False
return True
def getURL(self, url, data=None, filename=None):
"""Simple interface for retrieving the contents of a URL
and writing it to a file or returning it as stringIO.
"""
#sys.stderr.write('getURL %s\n' % url)
rsp = self.b.open(url, data)
if filename:
f = file("./Inputs/"+filename, 'wb') # always write to Input folder
f.write(rsp.read())
f.close()
return None
else:
return StringIO(rsp.read())
def putFileURL(self, filename, url,
fileobj=None, title=None, type='text/plain'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url,'createObject?type_name=File')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def getFile(self, url, filename=None):
""" getFile -- gets a file using standard download from Plone site
url: The URL is pointer to the file on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url,'at_download/file'))
if filename:
f = open(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
return rsp.read()
def saveFile(self, url, dir=".", at_download=False):
"""Reads the response from a URL and saves it to a local
file based on the name provided in the Content-Disposition
header.
The dir field specifies to the directory where the file will
be stored.
If the at_download flag is True then 'at_download/file' will
be appended the URL.
"""
if at_download:
rsp = self.b.open(url_join(url,'at_download/file'))
else:
rsp = self.b.open(url)
fname = get_filename(rsp)
if fname:
f = open('/'.join([dir, fname]), 'wb')
f.write(rsp.read())
f.close()
return fname
def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'):
sys.stderr.write('putImageURL %s to %s\n' % (imgfile,url))
self.b.open('/'.join((url,'createObject?type_name=Image')))
self.b.select_form("edit_form")
if not title: title = path.basename(imgfile)
self.b['title'] = title
self.b.add_file(open(imgfile), type, path.basename(imgfile))
try:
# doesn't seem necessary but it is required for file upload
form = self.b.find_control('image_delete')
form.value = ["",]
# print "This really does need to happen.."
except:
# print "That delete stuff never happens..."
pass
self.b.submit("form.button.save")
def putImageURL(self, filename, url,
fileobj=None, title=None, type='image/jpg'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url,'createObject?type_name=Image')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def putDocument(self, doc, url, title):
"""Creates a new document and add the doc (file-like object) to it."""
self.b.open('/'.join((url,'createObject?type_name=Document')))
self.b.select_form("edit_form")
self.b['title'] = title
doc.seek(0)
self.b['text'] = doc.read()
self.b.submit("form.button.save")
return self.b.geturl()
def getDocument(self, url):
"""Returns a string with the text of the current document."""
self.b.open('/'.join((url,'edit')))
self.b.select_form("edit_form")
s = self.b['text']
self.b.submit("form.button.cancel")
return s
def editDocument(self, doc, url, title=None):
"""Replaces the contents of a document"""
self.b.open('/'.join((url,'edit')))
self.b.select_form("edit_form")
# update the title if given
if title: self.b['title'] = title
# slightly dangerous approach where we seek first
# to test for a file-like object. If exception is
# thrown then assume doc is a string.
try:
doc.seek(0)
self.b['text'] = doc.read()
except:
self.b['text'] = doc
self.b.submit("form.button.save")
return self.b.geturl()
def createFolder(self, folder, url):
"""Creates a folder titled <folder> at the location <url> if
it doesn't already exist. Returns the full path of new folder.
"""
pathurl = '/'.join((url,folder.lower().replace(' ','-')))
print pathurl
try:
self.b.open(pathurl)
except:
self.b.open('/'.join((url,'createObject?type_name=Folder')))
self.b.select_form("edit_form")
self.b['title'] = folder
self.b.submit("form.button.save")
return self.b.geturl()
def editFolder(self, url, title="", description=""):
"""Edit the basic fields of the Folder. Mostly useful for
setting the title AFTER creating the folder with a reasonable
short name.
"""
try:
self.b.open(url)
except:
self.error = True
return None
self.b.open(url+'/edit')
self.b.select_form("edit_form")
if title:
self.b['title'] = title
if description:
self.b['description'] = description
self.b.submit("form.button.save")
return self.b.geturl()
def deleteFolder(self, url):
"Deletes folder and all of its contents"
sys.stderr.write('DELETING folder %s\n' % url)
return
# Puts SimMaps on to the site
def putSimMap(self, simmap, mapfile, url,
simmap_file=None, mapfile_file=None,
title=None, description=None,
trans=.7, details=None, zoom=8):
""" putSimMap
Required Input: simmap, mapfile, url
simmap is a file that contains the desired GIS layer
mapfile is the standard .map file that maps GIS layer to image
url - is the full URL to the folder where the SimMap will be stored
Optional Inputs: simmap_file, mapfile_file, title, trans, details
simmap_file - file-like object of the simmap if None simmap
will be openned and read.
mapfile_file - file-like object of the mapfile. If None mapfile
will be openned and read.
title - title sting of the SimMap defaults to basename(simmap)
trans - transparency level
details - description of SimMap as stored in the details field
"""
self.b.open('/'.join((url,'createObject?type_name=SimMap')))
self.b.select_form("edit_form")
if not simmap_file: simmap_file = open(simmap, 'rb')
if not mapfile_file: mapfile_file = open(mapfile, 'rb')
if not title: title = path.splitext(path.basename(simmap))[0]
self.b['title'] = str(title)
self.b.add_file(simmap_file, 'application/octet-stream',
path.basename(simmap), "simImage_file")
self.b.add_file(mapfile_file, 'application/octet-stream',
path.basename(mapfile), "mapFile_file")
if description: self.b['description'] = str(description)
self.b['transparency'] = str(trans)
self.b['zoom'] = str(zoom)
if details: self.b['details'] = str(details)
self.b.submit("form.button.save")
return self.b.geturl()
def getSimMap(self, url):
""" return the SimMap metadata
Gets the metadata associated with SimMap including title,
description, location, transparency, and zoom.
= """
self.b.open(url+'/edit')
self.b.select_form('edit_form')
d = dict(
title = self.b['title'],
description = self.b['description'],
details = self.b['details'],
location = self.b['location'],
transparency = self.b['Transparency'],
zoom = self.b['zoom'],
)
self.b.submit("form.button.cancel")
return d
def updateSimMap(self, url, **kwargs):
""" update the SimMap metadata
Keywords must match the field names from the edit form exactly,
extra keywords (or mispelled ones) will be ignored.
"""
self.b.open(url+'/edit')
self.b.select_form('edit_form')
for k in kwargs:
if k in self.b:
self.b[k] = str(kwargs[k])
self.b.submit("form.button.save")
def getSimMapData(self, url, filename=None):
""" getSimMapData -- gets the data component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
bufsize = 15 * 1024 * 1024
rsp = self.b.open(url_join(url,'at_download/simImage'))
if filename:
f = file(filename, 'wb')
while 1:
b = rsp.read(bufsize)
f.write(b)
if len(b) < bufsize: break
f.close()
return None
else:
return StringIO(rsp.read())
def getSimMapMapfile(self, url, filename=None):
""" getSimMapData -- gets the mapfile component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url,'at_download/mapFile'))
if filename:
f = file(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
data = StringIO()
data.write(rsp.read())
return data
def putProbmapCache(self, url, filename):
"""store precomputed probmaps as part of the Driver Set
This is a temporary method until the REST interface
is ready. It's really depricated before it written!
"""
self.b.open(url+'/edit')
self.b.select_form('edit_form')
with open(filename, 'rb') as f:
self.b.add_file(f, 'application/octet-stream',
path.basename(filename), name='probfile_file')
self.b.submit("form.button.save")
# def putAttrmapCache(self, url, filepathbasename, filename):
# """store precomputed maps as part of the Driver Set
# This is a temporary method until the REST interface
# is ready. It's really depricated before it written!
# """
# self.b.open(url+'/edit')
# self.b.select_form('edit_form')
# with open(filename, 'w') as f:
# self.b.add_file(f, 'application/octet-stream',
# path.basename(filepathbasename), name=filename)
# self.b.submit("form.button.save")
# DELETE FUNCTIONS -----------------------------------------
# These functions delete items from the LEAM Plone sites
# ----------------------------------------------------------
def deleteItem(self,fname,url):
""" deleteItem
Deletes <fname> from the folder <url>
"""
print '/'.join((url,fname,'delete_confirmation'))
print self.b.open('/'.join((url,fname,'delete_confirmation')))
self.b.select_form(None,None,1)
print self.b.submit()
def trilegal(outputFileName,
longitude = 0, latitude = 0,
coordinateType = "galactic",
fieldArea = 1,
passband = 4, magnitudeLimit = 26, magnitudeResolution = 0.1,
IMFtype = 3,
includeBinaries = True, binaryFraction = 0.3, lowerBinaryMassRatio = 0.7, upperBinaryMassRatio = 1.0,
extinctionType = 2, extinctionValue = 0.0378, extinctionSigma = 0.0,
useThinDisc = False,
useThickDisc = False,
useBulge = True):
"""
Query the web interface of the TRILEGAL population synthesis code.
The TRILEGAL webform is automatically filled and submitted. The computations are done locally
on Girardi's computer. As soon as they are finished, the script retrieves the data file.
Example:
>>> trilegal("output.txt", longitude=3, latitude=14, coordinateType="galactic", fieldArea=1, magnitudeLimit=7, useThinDisc=True)
@param outputFileName: name of file wherein trilegal output will be saved
@type outputFileName: string
@param longitude: galactic longitude (degrees) or right ascension (hours)
@type longitude: integer
@param latitude: galactic latitude (degrees) or declination (degrees)
@type latitude: integer
@param coordinateType: either "galactic", or "equatorial"
@type coordinateType: string
@param fieldArea: total field area in square degrees (max. 10 deg^2)
@type fieldArea: float
@param passband: U,B,V,R,I,J,H,K = 1,2,3,4,5,6,7,8 for magnitude limit
@type passband: integer
@param magnitudeLimit: magnitude limit in specified passband
@type magnitudeLimit: float
@param magnitudeResolution: Distance modulus resolution of Galaxy components (mag)
@type magnitudeResolution: float
@param IMFtype: type of Initial Mass Function of single stars
1 = Salpeter with cutoff at 0.01, Msun,
2 = Chabrier exponential,
3 = Chabrier lognormal,
4 = Kroupa corrected for binaries,
5 = Kroupa not corrected for binaries
@type IMFtype: integer
@param includeBinaries: include binaries in the population (True or False)
@type includeBinaries: boolean
@param binaryFraction: fraction of binaries
@type binaryFraction: float
@param lowerBinaryMassRatio: lower limit of binary mass fraction
@type lowerBinaryMassRatio: float
@param upperBinaryMassRatio: upper limit of binary mass fraction
@type upperBinaryMassRatio: float
@param extinctionType: Type of extinction
0: no dust extinction
1: local calibration
2: calibration at infinity
@type extinctionType: integer
@param extinctionValue: for a local calibration this is dAv/dr in mag/pc
for the calibration at infinity this is Av at infinity in mag.
@type extinctionValue: float
@param extinctionSigma: 1-sigma extinction dispersion / total extinction (max. 0.3)
@type extinctionSigma: float
@param useThinDisk: if True use squared hyperbolic secant along z, if False don't include
@type useThinDisk: boolean
@param useThickDisk: if True use squared hyperbolic secant along z, if False don't include
@type useThickDisk: boolean
@param useBulge: if True use triaxal bulge, if False don't include
@type useBulge: boolean
@return None. A file is retrieved
"""
# The latest Trilegal web version
trilegalURL = "http://stev.oapd.inaf.it/cgi-bin/trilegal"
# Get the web form
timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime())
print("{0}: Opening TRILEGAL web interface".format(timestamp))
myBrowser = Browser()
try:
myBrowser.open(trilegalURL)
except:
timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime())
print("{0}: Unable to open the TRILEGAL website".format(timestamp))
return
myBrowser.select_form(nr=0) # there is only one form...
# Fill in the form. To know how the different fields in the form are
# named, we used
# >>> request = mechanize.Request(trilegalURL)
# >>> response = mechanize.urlopen(request)
# >>> forms = mechanize.ParseResponse(response, backwards_compat=False)
# >>> print forms[0]
timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime())
print("{0}: Filling TRILEGAL web form".format(timestamp))
if coordinateType == "galactic":
myBrowser["gal_coord"] = ["1"]
myBrowser["gc_l"] = str(longitude)
myBrowser["gc_b"] = str(latitude)
else:
myBrowser["gal_coord"] = ["2"]
myBrowser["eq_alpha"] = str(longitude)
myBrowser["eq_delta"] = str(latitude)
myBrowser["field"] = str(fieldArea)
myBrowser["icm_lim"] = str(passband)
myBrowser["mag_lim"] = str(magnitudeLimit)
myBrowser["mag_res"] = str(magnitudeResolution)
myBrowser["binary_kind"] = [str(int(includeBinaries))]
myBrowser["binary_frac"] = str(binaryFraction)
myBrowser["binary_mrinf"] = str(lowerBinaryMassRatio)
myBrowser["binary_mrsup"] = str(upperBinaryMassRatio)
myBrowser["extinction_kind"] = [str(extinctionType)]
if extinctionType == 1:
myBrowser["extinction_rho_sun"] = str(extinctionValue)
if extinctionType == 2:
myBrowser["extinction_infty"] = str(extinctionValue)
myBrowser["extinction_sigma"] = str(extinctionSigma)
if useThinDisc:
myBrowser["thindisk_kind"] = ["3"]
else:
myBrowser["thindisk_kind"] = ["0"]
if useThickDisc:
myBrowser["thickdisk_kind"] = ["3"]
else:
myBrowser["thickdisk_kind"] = ["0"]
if useBulge:
myBrowser["bulge_kind"] = ["2"]
else:
myBrowser["bulge_kind"] = ["0"]
# Submit the completed form
timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime())
print("{0}: Submitting completed TRILEGAL web form".format(timestamp))
nextWebPage = myBrowser.submit()
# Trilegal is now computing the result. Click on the special "Refresh"
# button until the webpage says that the computations are finished.
timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime())
print ("{0}: Waiting until TRILEGAL computations are finished".format(timestamp))
myBrowser.select_form(nr=0) # one form on the "be patient" web page
message = "Your job was finished"
while (message not in nextWebPage.read()):
nextWebPage = urlopen(myBrowser.click()) # click on the Refresh button
myBrowser.select_form(nr=0) # select form again, so that we can make a click again
sleep(5) # to not overload the website with refresh requests
# Get the url of the outputfile, and retrieve it. This can take a while.
timestamp = strftime("%a, %d %b %Y %H:%M:%S", gmtime())
print("{0}: Retrieving TRILEGAL output file".format(timestamp))
outputLink = myBrowser.links(url_regex="lgirardi/tmp/output").next()
urlretrieve(outputLink.absolute_url, outputFileName)
myBrowser.close()
# Save the parameters in an info file
parameterInfo = """
coordinateType {0}
longitude {1}
latitude {2}
fieldArea {3}
passband {4}
magnitudeLimit {5}
magnitudeResolution {6}
IMFtype {7}
includeBinaries {8}
binaryFraction {9}
lowerBinaryMassRatio {10}
upperBinaryMassRatio {11}
extinctionType {12}
extinctionValue {13}
extinctionSigma {14}
""".format(coordinateType,
longitude,
latitude,
fieldArea,
passband,
magnitudeLimit,
magnitudeResolution,
IMFtype,
includeBinaries,
binaryFraction,
lowerBinaryMassRatio,
upperBinaryMassRatio,
extinctionType,
extinctionValue,
extinctionSigma)
infoFileName = "info_" + outputFileName
with open(infoFileName, 'w') as infoFile:
infoFile.write(parameterInfo)
class LEAMsite:
def __init__(self, site, user, passwd):
self.site = site
self.error = False
self.b = Browser()
self.b.set_handle_robots(False)
try:
self.b.open(site)
except urllib2.URLError:
self.error = True
return
try:
# try and log in from the portlet
self.b.select_form('loginform')
except:
# try logging in from the main login page
self.b.open('/'.join((site, "login_form")))
self.b.select_form(nr=1)
if not user or not passwd:
raise ValueError('user and password are required')
self.b['__ac_name'] = user
self.b['__ac_password'] = passwd
r = self.b.open(self.b.click())
# plone changes have rendered this inoperable
# capture the response and look in the content
#
#if 'logged_in' in path.split(self.b.geturl()):
# sys.stderr.write("Error: unable to login to LEAM Plone site\n")
# sys.exit(1)
def getURL(self, url, data=None, filename=None):
"""Simple interface for retrieving the contents of a URL
and writing it to a file or returning it as stringIO.
"""
#sys.stderr.write('getURL %s\n' % url)
rsp = self.b.open(url, data)
if filename:
f = file(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
return StringIO(rsp.read())
def putFileURL(self,
filename,
url,
fileobj=None,
title=None,
type='text/plain'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url, 'createObject?type_name=File')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def getFile(self, url, filename=None):
""" getFile -- gets a file using standard download from Plone site
url: The URL is pointer to the file on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url, 'at_download/file'))
if filename:
f = open(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
return rsp.read()
def saveFile(self, url, dir=".", at_download=False):
"""Reads the response from a URL and saves it to a local
file based on the name provided in the Content-Disposition
header.
The dir field specifies to the directory where the file will be stored.
If the at_download flag is True then 'at_download/file' will
be appended the URL.
"""
if at_download:
rsp = self.b.open(url_join(url, 'at_download/file'))
else:
rsp = self.b.open(url)
fname = get_filename(rsp)
if fname:
f = open('/'.join([dir, fname]), 'wb')
f.write(rsp.read())
f.close()
return fname
def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'):
sys.stderr.write('putImageURL %s to %s\n' % (imgfile, url))
self.b.open('/'.join((url, 'createObject?type_name=Image')))
self.b.select_form("edit_form")
if not title: title = path.basename(imgfile)
self.b['title'] = title
self.b.add_file(open(imgfile), type, path.basename(imgfile))
try:
# doesn't seem necessary but it is required for file upload
form = self.b.find_control('image_delete')
form.value = [
"",
]
# print "This really does need to happen.."
except:
# print "That delete stuff never happens..."
pass
self.b.submit("form.button.save")
def putImageURL(self,
filename,
url,
fileobj=None,
title=None,
type='image/jpg'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url, 'createObject?type_name=Image')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def putDocument(self, doc, url, title):
"""Creates a new document and add the doc (file-like object) to it."""
self.b.open('/'.join((url, 'createObject?type_name=Document')))
self.b.select_form("edit_form")
self.b['title'] = title
doc.seek(0)
self.b['text'] = doc.read()
self.b.submit("form.button.save")
return self.b.geturl()
def getDocument(self, url):
"""Returns a string with the text of the current document."""
self.b.open('/'.join((url, 'edit')))
self.b.select_form("edit_form")
s = self.b['text']
self.b.submit("form.button.cancel")
return s
def editDocument(self, doc, url, title=None):
"""Replaces the contents of a document"""
self.b.open('/'.join((url, 'edit')))
self.b.select_form("edit_form")
# update the title if given
if title: self.b['title'] = title
# slightly dangerous approach where we seek first
# to test for a file-like object. If exception is
# thrown then assume doc is a string.
try:
doc.seek(0)
self.b['text'] = doc.read()
except:
self.b['text'] = doc
self.b.submit("form.button.save")
return self.b.geturl()
def createFolder(self, folder, url):
"""Creates a folder titled <folder> at the location <url> if
it doesn't already exist. Returns the full path of new folder.
"""
pathurl = '/'.join((url, folder.lower().replace(' ', '-')))
try:
self.b.open(pathurl)
except:
self.b.open('/'.join((url, 'createObject?type_name=Folder')))
self.b.select_form("edit_form")
self.b['title'] = folder
self.b.submit("form.button.save")
return self.b.geturl()
def editFolder(self, url, title="", description=""):
"""Edit the basic fields of the Folder. Mostly useful for
setting the title AFTER creating the folder with a reasonable
short name.
"""
try:
self.b.open(url)
except:
self.error = True
return None
self.b.open(url + '/edit')
self.b.select_form("edit_form")
if title:
self.b['title'] = title
if description:
self.b['description'] = description
self.b.submit("form.button.save")
return self.b.geturl()
def deleteFolder(self, url):
"Deletes folder and all of its contents"
sys.stderr.write('DELETING folder %s\n' % url)
return
# Puts SimMaps on to the site
def putSimMap(self,
simmap,
mapfile,
url,
simmap_file=None,
mapfile_file=None,
title=None,
description=None,
trans=.7,
details=None,
zoom=11):
""" putSimMap
Required Input: simmap, mapfile, url
simmap is a file that contains the desired GIS layer
mapfile is the standard .map file that maps GIS layer to image
url - is the full URL to the folder where the SimMap will be stored
Optional Inputs: simmap_file, mapfile_file, title, trans, details
simmap_file - file-like object of the simmap if None simmap
will be openned and read.
mapfile_file - file-like object of the mapfile. If None mapfile
will be openned and read.
title - title sting of the SimMap defaults to basename(simmap)
trans - transparency level
details - description of SimMap as stored in the details field
"""
self.b.open('/'.join((url, 'createObject?type_name=SimMap')))
self.b.select_form("edit_form")
if not simmap_file: simmap_file = open(simmap, 'rb')
if not mapfile_file: mapfile_file = open(mapfile, 'rb')
if not title: title = path.splitext(path.basename(simmap))[0]
self.b['title'] = str(title)
self.b.add_file(simmap_file, 'application/octet-stream',
path.basename(simmap), "simImage_file")
self.b.add_file(mapfile_file, 'application/octet-stream',
path.basename(mapfile), "mapFile_file")
if description: self.b['description'] = str(description)
self.b['transparency'] = str(trans)
self.b['zoom'] = str(zoom)
if details: self.b['details'] = str(details)
self.b.submit("form.button.save")
return self.b.geturl()
def getSimMap(self, url):
""" getSimMap
Gets the metadata associated with SimMap including title,
description, location, transparency, and zoom.
"""
self.b.open(url + '/edit')
self.b.select_form('edit_form')
d = dict(
title=self.b['title'],
description=self.b['description'],
details=self.b['details'],
location=self.b['location'],
transparency=self.b['Transparency'],
zoom=self.b['zoom'],
)
self.b.submit("form.button.cancel")
return d
def getSimMapData(self, url, filename=None):
""" getSimMapData -- gets the data component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
bufsize = 15 * 1024 * 1024
rsp = self.b.open(url_join(url, 'get_layer'))
if filename:
f = file(filename, 'wb')
while 1:
b = rsp.read(bufsize)
f.write(b)
if len(b) < bufsize: break
f.close()
return None
else:
return StringIO(rsp.read())
def getSimMapMapfile(self, url, filename=None):
""" getSimMapData -- gets the mapfile component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url, 'get_mapfile'))
if filename:
f = file(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
data = StringIO()
data.write(rsp.read())
return data
# DELETE FUNCTIONS -----------------------------------------
# These functions delete items from the LEAM Plone sites
# ----------------------------------------------------------
def deleteItem(self, fname, url):
""" deleteItem
Deletes <fname> from the folder <url>
"""
print '/'.join((url, fname, 'delete_confirmation'))
print self.b.open('/'.join((url, fname, 'delete_confirmation')))
self.b.select_form(None, None, 1)
print self.b.submit()
from bs4 import BeautifulSoup
from mechanize import Browser
import re
import mechanize
url = "http://127.0.0.1/webgoat/attack"
br = Browser()
br.set_handle_robots(False)
br.add_password(url, "guest", "guest")
br.open(url)
br.select_form(nr=0)
sub = br.click(type="submit", nr=0)
br.open(sub)
soup = BeautifulSoup(br.response().read(), 'lxml')
link = soup.a
all_links = link.find_all_next('a')
print "Welche SQL-Operation?\n"
i = 0
SQL_links = []
for link in all_links:
if re.search(r'menu=1200', str(link)):
i = i + 1
SQL_links.append(str(link['href']))
def attack(address, htmlObject):
br = Browser()
br.open(address)
br.click(htmlObject)
def attack(address, htmlObject):
br = Browser()
br.open(address)
br.click(htmlObject)
