def parsebody(body):
global sessstr,pagetype
body=body.replace("%2d","-")
body=body.replace("%2D","-")
body=body.replace("%25","%")
body=body.replace("%20"," ")
body=body.replace("+"," ")
body=body.replace("%2b","+")
body=body.replace("%2B","+")
body=body.replace("%22","\"")
body=body.replace("\\\"","\"")
body=body.replace("\\'","'")
body=body.replace("\n","")
body=body.replace("\r","")
body=sre.sub(sessstr,"",body)
# These might cause problems
body=sre.sub("\<script([^\>]*)\>(.*?)\</script\>","",body)
body=sre.sub("\<!--(.*?)--\>","",body)
if pagetype=="php":
body=sre.sub("(?i)\<input type=\"hidden\" name=\"phpsessid\" value=\"([a-z0-9]{32})\"( /|)\>","",body)
body=sre.sub(" alt=(\"|\')([^\\1\>]*)\\1","",body)
body=removeurls(body)
return body
def parsebody(body):
global sessstr, pagetype
body = body.replace("%2d", "-")
body = body.replace("%2D", "-")
body = body.replace("%25", "%")
body = body.replace("%20", " ")
body = body.replace("+", " ")
body = body.replace("%2b", "+")
body = body.replace("%2B", "+")
body = body.replace("%22", "\"")
body = body.replace("\\\"", "\"")
body = body.replace("\\'", "'")
body = body.replace("\n", "")
body = body.replace("\r", "")
body = sre.sub(sessstr, "", body)
# These might cause problems
body = sre.sub("\<script([^\>]*)\>(.*?)\</script\>", "", body)
body = sre.sub("\<!--(.*?)--\>", "", body)
if pagetype == "php":
body = sre.sub(
"(?i)\<input type=\"hidden\" name=\"phpsessid\" value=\"([a-z0-9]{32})\"( /|)\>",
"", body)
body = sre.sub(" alt=(\"|\')([^\\1\>]*)\\1", "", body)
body = removeurls(body)
return body
def remotePath(self):
log.debug( '> domain.ProgramFromQuery.remotePath()' )
remoteFile = self.basename()
remotePath = self.settings.get("paths_recordedprefix")
shostIp = self.settings.get( "mythtv_host" )
host = self.hostname()
try:
hostIp = socket.gethostbyname(host)
snet = hostIp[:6]
ssnet = shostIp[:6]
log.debug(snet + " " + ssnet)
# if the first 6 characters of both addresses don't match then
# chances are the 'internet' address of the host has been
# found so just use the setting IP
if snet <> ssnet:
hostIp = shostIp
except:
hostIp = shostIp
log.debug ('Host: %s hostIp: %s'%(host, hostIp))
# we should do this substitute anyway?!
remotePath = sre.sub( '\%h', hostIp, remotePath )
if sre.match( "^[Ss][Mm][Bb]:", remotePath ) and not sre.match( "^\d", host ):
host = sre.sub( '\..*$', '', host )
remotePath = string.replace( remotePath, '\\', '/' )
if remotePath[len(remotePath)-1] != '/':
remotePath += '/'
remotePath += remoteFile
log.debug('< domain.ProgramFromQuery.remotePath() => [%s]'%remotePath)
return str(remotePath)
def GetScanString(self, feet = True, punct = False, sylsOnly = False):
"""Return fully spaced line of all scansion marks (with options)"""
s = ''.join(self.charlist)
if not feet or sylsOnly: s = sre.sub('\|', ' ', s)
if not punct or sylsOnly: s = sre.sub('[-.,;:?!\(\)\"\']', ' ', s)
if sylsOnly: s = sre.sub('[^ ]', SYLMARK, s)
return s
def main():
gaia = wikipedia.getSite(code=u'en', fam=u'gaia')
plot = wikipedia.getSite(code=u'en', fam=u'plotwiki')
wikipedia.setAction(wikipedia.translate(gaia, msg))
wikipedia.setAction(wikipedia.translate(plot, msg))
final = u'<noinclude><!-- Do not edit this page, this page is automatically created by a Bot. -->\n'
final += u'==Most Recent Events==</noinclude>\n'
nonrecent = u'<noinclude>==Older Events==\n'
end = u'\n\'\'View everything here on the [[Plotwiki:|plotwiki...]]\'\'</noinclude>'
moreYears = True
year = 04
events = []
temp = []
while moreYears:
y = str(year)
page = wikipedia.Page(plot, u'Template:Pnav%s' % y.zfill(2))
try:
text = page.get()
r = sre.compile(u'^.*<span style=".*normal.*">(.*)</span>.*$',
sre.UNICODE | sre.MULTILINE | sre.DOTALL)
text = sre.sub(r, u'\\1', text)
r = sre.compile(u'\s+\|\s+',
sre.UNICODE | sre.MULTILINE | sre.DOTALL)
pages = sre.split(r, text)
r = sre.compile(u'\[\[([^|]*)(\|.*)?\]\]', sre.UNICODE)
for p in pages:
temp.append(sre.sub(r, u'\\1', p))
year += 1
except wikipedia.NoPage:
moreYears = False
for e in temp:
if not e in events:
events.append(e)
events = reversed(list(events))
x = 1
for e in events:
final += u'* [[Plotwiki:%s|]]\n' % e
x += 1
if x == 6:
final += nonrecent
if x <= 6:
final += end
final += end
page = wikipedia.Page(gaia, u'Plotwiki Current Events')
page.put(final)
def getattrval(body,attr):
body=sre.sub("([^\>]*)\>([^\000]*)","\\1",body)
if sre.search(attr+"=(\"|'|)([^\\1\ \>]*)\\1",body)!=None:
delim=sre.sub("[^\>]* "+attr+"=(\"|'|)([^\\1\ \>]*)\\1([^\>]*)","\\1",body)
exp="[^\>]* "+attr+"=(\\"+delim+")([^"
if delim=="": exp+="\ "
else: exp+=delim
exp+="\>]*)\\"+delim+"([^\>]*)"
return sre.sub(exp,"\\2",body)
else: return ""
def main(): gaia = wikipedia.getSite(code=u'en', fam=u'gaia') plot = wikipedia.getSite(code=u'en', fam=u'plotwiki') wikipedia.setAction(wikipedia.translate(gaia, msg)) wikipedia.setAction(wikipedia.translate(plot, msg)) final = u'<noinclude><!-- Do not edit this page, this page is automatically created by a Bot. -->\n' final+= u'==Most Recent Events==</noinclude>\n' nonrecent = u'<noinclude>==Older Events==\n' end = u'\n\'\'View everything here on the [[Plotwiki:|plotwiki...]]\'\'</noinclude>' moreYears = True year = 04 events = [] temp = [] while moreYears: y = str(year) page = wikipedia.Page(plot, u'Template:Pnav%s' % y.zfill(2)) try: text = page.get() r = sre.compile(u'^.*<span style=".*normal.*">(.*)</span>.*$', sre.UNICODE | sre.MULTILINE | sre.DOTALL) text = sre.sub(r, u'\\1', text) r = sre.compile(u'\s+\|\s+', sre.UNICODE | sre.MULTILINE | sre.DOTALL) pages = sre.split(r, text) r = sre.compile(u'\[\[([^|]*)(\|.*)?\]\]', sre.UNICODE) for p in pages: temp.append(sre.sub(r, u'\\1', p)) year+=1 except wikipedia.NoPage: moreYears = False for e in temp: if not e in events: events.append(e) events = reversed(list(events)); x = 1 for e in events: final+=u'* [[Plotwiki:%s|]]\n' % e x+=1 if x==6: final+=nonrecent if x<=6: final+=end final+=end page = wikipedia.Page(gaia, u'Plotwiki Current Events') page.put(final)
def getattrval(body, attr):
body = sre.sub("([^\>]*)\>([^\000]*)", "\\1", body)
if sre.search(attr + "=(\"|'|)([^\\1\ \>]*)\\1", body) != None:
delim = sre.sub("[^\>]* " + attr + "=(\"|'|)([^\\1\ \>]*)\\1([^\>]*)",
"\\1", body)
exp = "[^\>]* " + attr + "=(\\" + delim + ")([^"
if delim == "": exp += "\ "
else: exp += delim
exp += "\>]*)\\" + delim + "([^\>]*)"
return sre.sub(exp, "\\2", body)
else:
return ""
def retrieveurl(self, url):
waybackurl = 'http://web.archive.org/web/%s/%s' % (self.datestring,
url)
contents = urllib2.urlopen(waybackurl).read()
if contents.find(
"Sorry, we can't find the archived version of this page"
) != -1:
raise urllib2.URLError("not in wayback archive")
# remove the copyrighted javascript from the wayback machine...
contents = sre.sub(
'\\<SCRIPT language="Javascript"\\>(.|\r|\n)*(// FILE ARCHIVED ON [0-9]{14} AND RETRIEVED(.|\r|\n)* ON [0-9]{14}[.])(.|\r|\n)*\\</SCRIPT\\>',
'\\2', contents)
# replace the javascript-style comments indicating the retrieval with html comments
contents = sre.sub('// ((FILE|INTERNET).*)', '<!-- \\1 -->', contents)
return contents
def removeexplt(body,explt):
def removeit(body,explt): return body.replace(explt,"")
remove=lambda:removeit(body,explt)
try:
explt=str(int(explt))
body=sre.sub("(?<![0-9])"+explt+"(?![0-9])","",body)
except ValueError:
body=remove()
explt=parsebody(explt)
body=remove()
# if pagetype=="asp":
# explt=explt.replace("%2D","-")
# body=remove()
#
# explt=explt.replace("+"," ")
# body=remove()
#
# explt=explt.replace("%20"," ")
# body=remove()
#
# explt=explt.replace("%2b","+")
# body=remove()
# body=body.replace(explt.replace("+"," "),"")
# body=body.replace(explt.replace("%2D","-"),"")
# body=body.replace(explt.replace("+"," ").replace("%2D","-"),"")
return body
def __str__(self):
ret = ""
x = 0
for i in self.plain_iter():
ret = ret + ("%02i:\n%s\n" % (x, sre.sub(self._tabberpat, " ", str(i)))).strip() + "\n"
x += 1
return ret.strip()
def removeexplt(body, explt):
def removeit(body, explt):
return body.replace(explt, "")
remove = lambda: removeit(body, explt)
try:
explt = str(int(explt))
body = sre.sub("(?<![0-9])" + explt + "(?![0-9])", "", body)
except ValueError:
body = remove()
explt = parsebody(explt)
body = remove()
# if pagetype=="asp":
# explt=explt.replace("%2D","-")
# body=remove()
#
# explt=explt.replace("+"," ")
# body=remove()
#
# explt=explt.replace("%20"," ")
# body=remove()
#
# explt=explt.replace("%2b","+")
# body=remove()
# body=body.replace(explt.replace("+"," "),"")
# body=body.replace(explt.replace("%2D","-"),"")
# body=body.replace(explt.replace("+"," ").replace("%2D","-"),"")
return body
def exploit(url, qry, namval, explt, post, poststring="", randstr=""):
def removeexplt(body, explt):
def removeit(body, explt):
return body.replace(explt, "")
remove = lambda: removeit(body, explt)
try:
explt = str(int(explt))
body = sre.sub("(?<![0-9])" + explt + "(?![0-9])", "", body)
except ValueError:
body = remove()
explt = parsebody(explt)
body = remove()
# if pagetype=="asp":
# explt=explt.replace("%2D","-")
# body=remove()
#
# explt=explt.replace("+"," ")
# body=remove()
#
# explt=explt.replace("%20"," ")
# body=remove()
#
# explt=explt.replace("%2b","+")
# body=remove()
# body=body.replace(explt.replace("+"," "),"")
# body=body.replace(explt.replace("%2D","-"),"")
# body=body.replace(explt.replace("+"," ").replace("%2D","-"),"")
return body
explt = str(explt)
randstr = str(randstr)
if randstr != "" and explt != randstr:
explt += randstr # explt!=randstr possibly not needed
expltqry = sre.sub(namval + "[^\&]*(?=\&|$)", namval + explt, qry)
if not post: hab = getpage(url + expltqry, poststring=poststring)
else: hab = getpage(url, poststring=expltqry)
if listempty(hab): return
# headers=hab[0]
body = hab[1]
##### ? # sre for all this crap? #####
if len(explt) >= 4:
body = removeexplt(body, explt)
body = parsebody(body)
body = removeexplt(body, explt)
else:
body = parsebody(body)
if randstr != "": body = removeexplt(body, randstr)
return body
def decapitate(body):
if body == None: return
global decapitation
if decapitation == True: body = sre.sub("<head>(.+?)</head>", "", body)
return body
def decapitate(body):
if body==None: return
global decapitation
if decapitation==True: body=sre.sub("<head>(.+?)</head>","",body)
return body
def exploit(url,qry,namval,explt,post,poststring="",randstr=""):
def removeexplt(body,explt):
def removeit(body,explt): return body.replace(explt,"")
remove=lambda:removeit(body,explt)
try:
explt=str(int(explt))
body=sre.sub("(?<![0-9])"+explt+"(?![0-9])","",body)
except ValueError:
body=remove()
explt=parsebody(explt)
body=remove()
# if pagetype=="asp":
# explt=explt.replace("%2D","-")
# body=remove()
#
# explt=explt.replace("+"," ")
# body=remove()
#
# explt=explt.replace("%20"," ")
# body=remove()
#
# explt=explt.replace("%2b","+")
# body=remove()
# body=body.replace(explt.replace("+"," "),"")
# body=body.replace(explt.replace("%2D","-"),"")
# body=body.replace(explt.replace("+"," ").replace("%2D","-"),"")
return body
explt=str(explt)
randstr=str(randstr)
if randstr!="" and explt!=randstr: explt+=randstr # explt!=randstr possibly not needed
expltqry=sre.sub(namval+"[^\&]*(?=\&|$)",namval+explt,qry)
if not post: hab=getpage(url+expltqry,poststring=poststring)
else: hab=getpage(url,poststring=expltqry)
if listempty(hab): return
# headers=hab[0]
body=hab[1]
##### ? # sre for all this crap? #####
if len(explt)>=4:
body=removeexplt(body,explt)
body=parsebody(body)
body=removeexplt(body,explt)
else: body=parsebody(body)
if randstr!="": body=removeexplt(body,randstr)
return body
def main(): #Setup Familys for Wikia Involved anime = wikipedia.getSite(code=u'en', fam=u'anime') wikipedia.setAction(wikipedia.translate(anime, msg)) siteList = [] #Get Project Wiki Listing wikiaIds = [] page = wikipedia.Page(anime, u'Bots/Wiki', None, None, 4)#4=Project Namespace try: text = page.get() r = sre.compile(u'^.*<!-- \|\|START\|\| -->\n?', sre.UNICODE | sre.DOTALL) text = sre.sub(r, u'', text) r = sre.compile(u'\n?<!-- \|\|END\|\| -->.*$', sre.UNICODE | sre.DOTALL) text = sre.sub(r, u'', text) r = sre.compile(u'\n', sre.UNICODE | sre.MULTILINE | sre.DOTALL) wikilist = sre.split(r, text) for wiki in wikilist: if wiki != u'': wikiaIds.append(wiki) except wikipedia.NoPage: moreYears = False for wiki in wikiaIds: siteList.append(wikipedia.getSite(code=u'en', fam=wiki)) commonstart = u'@import "http://en.anime.wikia.com/index.php?title=MediaWiki:Anime-Common.css&action=raw&ctype=text/css";' monobookstart = u'@import "http://en.anime.wikia.com/index.php?title=MediaWiki:Anime-Monobook.css&action=raw&ctype=text/css";' for site in siteList: common = wikipedia.Page(site, u'Common.css', None, None, 8)#8=MediaWiki Namespace monobook = wikipedia.Page(site, u'Monobook.css', None, None, 8)#8=MediaWiki Namespace siteSource = u'' try: siteSource = sitePage.get() except wikipedia.NoPage: wikipedia.output(u'Site %s has no %s template, creating it' % (site, template)) if siteSource != templateSource: wikipedia.output(u'Site \'%s\' template status: Needs Updating' % site) wikipedia.output(u'Updating template on %s' % site) sitePage.put(templateSource) else: wikipedia.output(u'Site \'%s\' template status: Ok' % site)
def rename_page(env, oldname, newname, user, ip, debug=False, db=None):
"""Rename a wiki page from oldname to newname, using env as the environment."""
handle_commit = False
if not db:
db = env.get_db_cnx()
handle_commit = True
cursor = db.cursor()
sqlbase = ' FROM wiki w1, ' + \
'(SELECT name, MAX(version) AS VERSION FROM WIKI GROUP BY NAME) w2 ' + \
'WHERE w1.version = w2.version AND w1.name = w2.name '
sql = 'SELECT w1.version,w1.text' + sqlbase + 'AND w1.name = \'%s\'' % oldname
if debug: print "Running query '%s'" % sql
cursor.execute(sql)
row = cursor.fetchone()
if not row:
raise TracError, 'Page not found'
new_wiki_page = (newname,row[0]+1,int(time.time()),user,ip,row[1],'Name changed from %s to %s'%(oldname,newname),0)
# Create a new page with the needed comment
if debug: print "Inserting new page '%s'" % repr(new_wiki_page)
cursor.execute('INSERT INTO wiki (name,version,time,author,ipnr,text,comment,readonly) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)', new_wiki_page)
# Move all the old versions of the page
if debug: print "Moving all old versions of page"
cursor.execute('UPDATE wiki SET name=%s WHERE name=%s', (newname,oldname))
# Move any attachments that are on the page
if debug: print "Moving all attachments in database"
cursor.execute('UPDATE attachment SET id=%s WHERE type="wiki" AND id=%s', (newname,oldname))
if debug: print "Found %s attachments on that page" % cursor.rowcount
if cursor.rowcount > 0:
# Change the directory where the attachments are stored, if there were any
if debug: print "Moving all attachments on file system"
os.renames(os.path.join(env.path, 'attachments/wiki', oldname), os.path.join(env.path, 'attachments/wiki', newname))
# Get a list of all wiki pages containing links to the old page
if debug: print "Trying to fix links"
sql = 'SELECT w1.version,w1.name,w1.text' + sqlbase + "AND w1.text like '%%[wiki:%s%%'" % oldname
if debug: print "Running query '%s'" % sql
cursor.execute(sql)
# Rewrite all links to the old page, such as to point to the new page
for row in list(cursor):
if debug: print "Found a page with a backlink in it: %s (v%s)" % (row[1],row[0])
newtext = sre.sub('\[wiki:%s'%oldname,'[wiki:%s'%newname,row[2])
cursor.execute('UPDATE wiki SET text=%s WHERE name=%s AND version=%s', (newtext,row[1],row[0]))
if handle_commit:
db.commit()
def rename_page(oldname, newname, envpath, debug=False):
"""Rename a wiki page from oldname to newname, using env as the environment."""
env = Environment(envpath)
db = env.get_db_cnx()
cursor = db.cursor()
sqlbase = ' FROM wiki w1, ' + \
'(SELECT name, MAX(version) AS VERSION FROM WIKI GROUP BY NAME) w2 ' + \
'WHERE w1.version = w2.version AND w1.name = w2.name '
sql = 'SELECT w1.version,w1.text' + sqlbase + 'AND w1.name = \'%s\'' % oldname
if debug: print "Running query '%s'" % sql
cursor.execute(sql)
row = cursor.fetchone()
if not row:
raise TracError, 'Page not found'
remote_user = username()
new_wiki_page = (newname,row[0]+1,int(time.time()),remote_user,'127.0.0.1',row[1],'Name changed from %s to %s'%(oldname,newname),0)
# Create a new page with the needed comment
if debug: print "Inserting new page '%s'" % repr(new_wiki_page)
cursor.execute('INSERT INTO wiki (name,version,time,author,ipnr,text,comment,readonly) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)', new_wiki_page)
# Move all the old versions of the page
if debug: print "Moving all old versions of page"
cursor.execute('UPDATE wiki SET name=%s WHERE name=%s', (newname,oldname))
# Move any attachments that are on the page
if debug: print "Moving all attachments in database"
cursor.execute('UPDATE attachment SET id=%s WHERE type="wiki" AND id=%s', (newname,oldname))
if debug: print "Found %s attachments on that page" % cursor.rowcount
if cursor.rowcount > 0:
# Change the directory where the attachments are stored, if there were any
if debug: print "Moving all attachments on file system"
os.renames(os.path.join(envpath, 'attachments/wiki', oldname), os.path.join(envpath, 'attachments/wiki', newname))
# Get a list of all wiki pages containing links to the old page
if debug: print "Trying to fix links"
sql = 'SELECT w1.version,w1.name,w1.text' + sqlbase + "AND w1.text like '%%[wiki:%s%%'" % oldname
if debug: print "Running query '%s'" % sql
cursor.execute(sql)
# Rewrite all links to the old page, such as to point to the new page
for row in list(cursor):
if debug: print "Found a page with a backlink in it: %s (v%s)" % (row[1],row[0])
newtext = sre.sub('\[wiki:%s'%oldname,'[wiki:%s'%newname,row[2])
cursor.execute('UPDATE wiki SET text=%s WHERE name=%s AND version=%s', (newtext,row[1],row[0]))
db.commit()
def mirror(self, url, datestring):
if url in self.urlmap:
return
else:
filename = self.url2filename(url)
fn = filename.split('.')
filename = '.'.join(fn[:-1])+'-'+datestring[:-6]+'.'+fn[-1]
if not self.testinclude(url):
return
self.urlmap[url] = filename
# TODO: add an option about re-reading stuff
if os.path.isfile(filename):
contents = open(filename, 'r').read()
else:
try:
time.sleep(1)
contents = self.retrieveurl(url)
except urllib2.URLError, e:
print 'could not retrieve url %s: %s' % (url, e)
return
self.ensuredir(os.path.dirname(filename))
linkscanner = HTMLLinkScanner()
try:
linkscanner.feed(contents)
except:
print 'could not parse %s as html' % url
linkstomirror = []
for link in linkscanner.links:
linkurl = urlparse.urljoin(url, link)
linkstomirror.append(linkurl)
contents = sre.sub('http://web.archive.org/web/[0-9]{14}/', '', contents)
for tagtext, attr, link in linkscanner.replacements:
scheme, location, path, query, fragment = urlparse.urlsplit(link)
newtext = None
if tagtext.lower().startswith('<base'):
# strip out base references
newtext = ''
elif scheme or location:
if not self.testinclude(link): continue
linkfilename = self.url2filename(link)
newtext = tagtext.replace(link, 'file://%s' % linkfilename)
elif path.startswith('/'):
linkurl = urlparse.urljoin(url, link)
linkfilename = self.url2filename(linkurl)
newtext = tagtext.replace(link, 'file://%s' % linkfilename)
if newtext is not None:
contents = contents.replace(tagtext, newtext)
contentsfile = open(filename, 'w')
contentsfile.write(contents)
contentsfile.close()
for linkurl in linkstomirror:
self.mirror(linkurl)
def mirror(self, url):
if url in self.urlmap:
return
else:
filename = self.url2filename(url)
if not self.testinclude(url):
return
print url, '->', filename
self.urlmap[url] = filename
# TODO: add an option about re-reading stuff
if os.path.isfile(filename):
contents = open(filename, 'r').read()
else:
try:
contents = self.retrieveurl(url)
except urllib2.URLError, e:
print 'could not retrieve url %s: %s' % (url, e)
return
self.ensuredir(os.path.dirname(filename))
linkscanner = HTMLLinkScanner()
try:
linkscanner.feed(contents)
except:
print 'could not parse %s as html' % url
linkstomirror = []
for link in linkscanner.links:
linkurl = urlparse.urljoin(url, link)
linkstomirror.append(linkurl)
contents = sre.sub('http://web.archive.org/web/[0-9]{14}/', '',
contents)
for tagtext, attr, link in linkscanner.replacements:
scheme, location, path, query, fragment = urlparse.urlsplit(
link)
newtext = None
if tagtext.lower().startswith('<base'):
# strip out base references
newtext = ''
elif scheme or location:
if not self.testinclude(link): continue
linkfilename = self.url2filename(link)
newtext = tagtext.replace(link, 'file://%s' % linkfilename)
elif path.startswith('/'):
linkurl = urlparse.urljoin(url, link)
linkfilename = self.url2filename(linkurl)
newtext = tagtext.replace(link, 'file://%s' % linkfilename)
if newtext is not None:
contents = contents.replace(tagtext, newtext)
contentsfile = open(filename, 'w')
contentsfile.write(contents)
contentsfile.close()
for linkurl in linkstomirror:
self.mirror(linkurl)
def fullpath(url, pageurl):
if url.lower()[:7] == "http://": return url
if pageurl.count("?") != 0: pageurl = pageurl[:pageurl.find("?")]
if url.count("?") > 0:
if url[:1] == "?": return pageurl + url
pageurl = pageurl[:pageurl.find("?")]
#pageurl=pageurl[:pageurl.find("?")]
pagedomain = pageurl[:pageurl[7:].find("/") + 7]
if url[:1] == "/": return pagedomain + url
pagepath = pageurl[pageurl[7:].find("/") + 7:]
pagepath = pagepath[:pagepath.rfind("/") + 1]
path = pagepath + url
path = sre.sub("\.\/", "\/", path)
path = sre.sub("\/([^\/]+)\/..\/", "\/", path)
return pagedomain + path
def fullpath(url,pageurl):
if url.lower()[:7]=="http://": return url
if pageurl.count("?")!=0: pageurl=pageurl[:pageurl.find("?")]
if url.count("?")>0:
if url[:1]=="?": return pageurl+url
pageurl=pageurl[:pageurl.find("?")]
#pageurl=pageurl[:pageurl.find("?")]
pagedomain=pageurl[:pageurl[7:].find("/")+7]
if url[:1]=="/": return pagedomain+url
pagepath=pageurl[pageurl[7:].find("/")+7:]
pagepath=pagepath[:pagepath.rfind("/")+1]
path=pagepath+url
path=sre.sub("\.\/","\/",path)
path=sre.sub("\/([^\/]+)\/..\/","\/",path)
return pagedomain+path
def dbTime2MythTime(dt):
"""
Converts a time from the database into a format MythTV understands.
@param dt: a datetime, timedelta, or string time returned from the pure python mysql module
@return: time in mythtv format as a string
"""
if type(dt) is str:
# TODO: Remove me - pure python mysql support
s = sre.sub( "[T:|\\-| ]", "", dt)
elif type(dt) is datetime.datetime:
# Native MySQLdb
s = dt.strftime("%Y%m%d%H%M%S")
elif type(dt) is datetime.timedelta:
s = time.strftime("%H%M%S",time.gmtime(dt.seconds))
elif type(dt) is datetime.date:
s = dt.strftime("%Y%m%d")
else:
raise Exception, 'cannot convert time of unexpected type %s' % type(dt)
return s
raise TestFailed, "sre.search"
try:
assert sre.match('a*', 'xxx').span(0) == (0, 0)
assert sre.match('a*', 'xxx').span() == (0, 0)
assert sre.match('x*', 'xxxa').span(0) == (0, 3)
assert sre.match('x*', 'xxxa').span() == (0, 3)
assert sre.match('a+', 'xxx') == None
except:
raise TestFailed, "sre.search"
if verbose:
print 'Running tests on sre.sub'
try:
assert sre.sub("(?i)b+", "x", "bbbb BBBB") == 'x x'
def bump_num(matchobj):
int_value = int(matchobj.group(0))
return str(int_value + 1)
assert sre.sub(r'\d+', bump_num, '08.2 -2 23x99y') == '9.3 -3 24x100y'
assert sre.sub(r'\d+', bump_num, '08.2 -2 23x99y', 3) == '9.3 -3 23x99y'
assert sre.sub('.', lambda m: r"\n", 'x') == '\\n'
assert sre.sub('.', r"\n", 'x') == '\n'
s = r"\1\1"
assert sre.sub('(.)', s, 'x') == 'xx'
assert sre.sub('(.)', sre.escape(s), 'x') == s
assert sre.sub('(.)', lambda m: s, 'x') == s
def rename_page(env, oldname, newname, user, ip, debug=False, db=None):
"""Rename a wiki page from oldname to newname, using env as the environment."""
handle_commit = False
if not db:
db = env.get_db_cnx()
handle_commit = True
cursor = db.cursor()
if debug is False:
debug = pass_
elif debug is True:
debug = print_
else:
debug = log_intercept(debug)
sqlbase = ' FROM wiki w1, ' + \
'(SELECT name, MAX(version) AS max_version FROM wiki GROUP BY name) w2 ' + \
'WHERE w1.version = w2.max_version AND w1.name = w2.name '
sql = 'SELECT w1.version,w1.text' + sqlbase + 'AND w1.name = \'%s\'' % oldname
debug('Running query %r', sql)
cursor.execute(sql)
row = cursor.fetchone()
if not row:
raise TracError, 'Page not found'
new_wiki_page = (newname,row[0]+1,int(time.time()),user,ip,row[1],'Name changed from %s to %s'%(oldname,newname),0)
# Create a new page with the needed comment
debug('Inserting new page %r', new_wiki_page)
cursor.execute('INSERT INTO wiki (name,version,time,author,ipnr,text,comment,readonly) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)', new_wiki_page)
# Move all the old versions of the page
debug("Moving all old versions of page")
cursor.execute('UPDATE wiki SET name=%s WHERE name=%s', (newname,oldname))
# Move any attachments that are on the page
debug("Moving all attachments in database")
cursor.execute('UPDATE attachment SET id=%s WHERE type=%s AND id=%s', (newname,'wiki',oldname))
debug("Found %s attachments on that page", cursor.rowcount)
if cursor.rowcount > 0:
# Change the directory where the attachments are stored, if there were any
debug('Moving all attachments on file system')
from_path = os.path.join(env.path, 'attachments', 'wiki', urllib.quote(oldname))
to_path = os.path.join(env.path, 'attachments', 'wiki', urllib.quote(newname))
debug('Moving from %r to %r', from_path, to_path)
os.renames(from_path, to_path)
# Get a list of all wiki pages containing links to the old page
debug("Trying to fix links")
sql = 'SELECT w1.version,w1.name,w1.text' + sqlbase + "AND w1.text like '%%[wiki:%s%%'" % oldname
debug('Running query %r', sql)
cursor.execute(sql)
# Rewrite all links to the old page, such as to point to the new page
for row in list(cursor):
debug("Found a page with a backlink in it: %s (v%s)", row[1], row[0])
newtext = sre.sub('\[wiki:%s'%oldname,'[wiki:%s'%newname,row[2])
cursor.execute('UPDATE wiki SET text=%s WHERE name=%s AND version=%s', (newtext,row[1],row[0]))
if handle_commit:
db.commit()
#! /usr/bin/python import os import sre import sys if len(sys.argv)<1: raise "pls input some path!" sys.argv=sys.argv[1:] print "input arguments is %s"%sys.argv for i in range(0,len(sys.argv),1): print "dir is %s"%sys.argv[i] if not os.path.isdir(sys.argv[i]): raise "pls input a valid path!" flist = os.listdir(sys.argv[i]) print "fileslist is %s"%flist for f in range(0,len(flist),1): print "filename is: %s"%flist[f] newname = sre.sub(r' +','_',flist[f]) print newname,flist[f] print "rename file..." os.rename(sys.argv[i]+os.sep+flist[f],sys.argv[i]+os.sep+newname)
def getpage(url, dheaders=1, redir=0, realpage=0, poststring="", exceptions=0):
# function to recurse and try getpage() again with new values
def recurse(exceptions):
sock.close()
exceptions += 1
if exceptions <= 6:
return getpage(url, dheaders, redir, realpage, poststring,
exceptions)
else:
print "Too many recursions, skipping..."
return
global usecookies, urllogfile, debug, ignorefileext
if not checkserver(servername(url)): return
if url.find("#") != -1: url = url[:url.find("#")]
# file extensions that need to be ignored code
fileext = sre.sub(".*(http\://[^/]*/).*", "\\1", url)
if url == fileext: fileext = "None"
else: fileext = sre.sub("^.*\/[^/]*\.([^\&\#\?\/]*)[^/]*$", "\\1", url)
if ignorefileext.count("," + fileext + ",") != 0: return
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((servername(url, False), 80))
workurl = pagename(url)
theurl = url
if redir != 1: theurl = workurl
qrytype = "GET"
if poststring != "": qrytype = "POST"
out = (qrytype + " " + theurl + " HTTP/1.1\n"
"Host: " + servername(url, False) + "\n"
"Connection: close\n")
if usecookies:
global cookies
out += "Cookie: " + cookies + "\n"
if poststring != "":
out += "Content-Type: application/x-www-form-urlencoded\n"
out += "Content-Length: " + str(len(poststring))
out += "\n\n" + poststring + "\n"
out += "\r\n\r\n"
sock.send(out)
# get response type and log the page
response = sock.recv(12)[-3:]
fp = open("logs/" + urllogfile, "a")
fp.write(url + ": " + response + " " + str(realpage) + "\n")
if poststring != "": fp.write(indent + "POST: " + poststring + "\n")
fp.close()
# at 404 response, close connection and fail
if response == "404" or response == "500":
sock.close()
return
# at 30[1237] response types, recurse new page
if sre.search("30[1237]", response):
while 1:
chunk = ""
byte = sock.recv(1)
while byte != "\r":
chunk += byte
byte = sock.recv(1)
sock.recv(1)
if chunk.lower()[:9] == "location:":
location = chunk.lower()[9:].strip()
if location == "http://" + servername(url, False) + url:
location = "/"
locpage = fullpath(location, url)
sock.close()
# if url[len(url)-2:]=="" and locpage[len(locpage)-4:]=="": break
redir = 1
if locpage != url:
redir = 0
if pagename(sre.sub("\\\\(\"|\')", "\\1",
locpage)) == pagename(url):
print "QUOTE REDIR"
return
print "OLD:", url
print "NEW:", chunk.lower()
print "REDIR:", locpage
return getpage(locpage, redir=redir, realpage=realpage)
if realpage == 1:
sock.close()
return
elif realpage == 1:
sock.close()
return url
# get headers, ignoring certain HTTP headers
headers = ""
type = 0
while 1:
chunk = ""
byte = sock.recv(1)
if byte == "\r":
sock.recv(1)
break
while byte != "\r":
chunk += byte
byte = sock.recv(1)
sock.recv(1)
if chunk.lower()[:11] != "set-cookie:" and chunk.lower(
)[:5] != "date:" and chunk.lower(
)[:15] != "content-length:" and chunk.lower(
)[:11] != "keep-alive:" and chunk.lower(
)[:18] != "transfer-encoding:" and chunk.lower(
)[:11] != "connection:":
headers += chunk
# if chunk.lower()[:15]=="content-length:":
# type=1
# conlen=int(chunk[16:])
if chunk.lower()[:26] == "transfer-encoding: chunked": type = 2
# no special type specified, just get the page
if type == 0:
body = ""
while 1:
chunk = sock.recv(200)
body += chunk
if chunk == "": break
# set it up if it does have a type
# else:
# byte=sock.recv(1)
# if byte=="\r": sock.recv(1)
# else:
# while 1:
# i=-1
# while byte!="\r":
# i+=1
# byte=sock.recv(1)
# nbytes=sock.recv(3)
# if nbytes=="\n\r\n": break
# # content-length
# if type==1:
# body=""
# for i in range(conlen):
# chunk=sock.recv(1)
# body+=chunk
# transfer-encoding: chunked
if type == 2:
body = ""
chunksize = ""
while chunksize != 0:
byte = ""
chunk = ""
while byte != "\r":
chunk += byte
byte = sock.recv(1)
sock.recv(1)
chunksize = int(chunk, 16)
wchunksz = chunksize
while wchunksz >= 1:
subchunk = sock.recv(wchunksz)
body += subchunk
wchunksz -= len(subchunk)
sock.recv(2)
# clean up and return
sock.close()
if dheaders != 1: headers = ""
return [headers, body, urlfix(url)]
# catch socket errors, such as "connection reset by peer" - trys again until it gives up and goes on to the next page
except socket.error:
print "Socket Error, Recursing..."
return recurse(exceptions)
def getpage(url,dheaders=1,redir=0,realpage=0,poststring="",exceptions=0):
# function to recurse and try getpage() again with new values
def recurse(exceptions):
sock.close()
exceptions+=1
if exceptions<=6: return getpage(url,dheaders,redir,realpage,poststring,exceptions)
else:
print "Too many recursions, skipping..."
return
global usecookies,urllogfile,debug,ignorefileext
if not checkserver(servername(url)): return
if url.find("#")!=-1: url=url[:url.find("#")]
# file extensions that need to be ignored code
fileext=sre.sub(".*(http\://[^/]*/).*","\\1",url)
if url==fileext: fileext="None"
else: fileext=sre.sub("^.*\/[^/]*\.([^\&\#\?\/]*)[^/]*$","\\1",url)
if ignorefileext.count(","+fileext+",")!=0: return
try:
sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((servername(url,False),80))
workurl=pagename(url)
theurl=url
if redir!=1: theurl=workurl
qrytype="GET"
if poststring!="": qrytype="POST"
out=(qrytype+" "+theurl+" HTTP/1.1\n"
"Host: "+servername(url,False)+"\n"
"Connection: close\n")
if usecookies:
global cookies
out+="Cookie: "+cookies+"\n"
if poststring!="":
out+="Content-Type: application/x-www-form-urlencoded\n"
out+="Content-Length: "+str(len(poststring))
out+="\n\n"+poststring+"\n"
out+="\r\n\r\n"
sock.send(out)
# get response type and log the page
response=sock.recv(12)[-3:]
fp=open("logs/"+urllogfile,"a")
fp.write(url+": "+response+" "+str(realpage)+"\n")
if poststring!="": fp.write(indent+"POST: "+poststring+"\n")
fp.close()
# at 404 response, close connection and fail
if response=="404" or response=="500":
sock.close()
return
# at 30[1237] response types, recurse new page
if sre.search("30[1237]",response):
while 1:
chunk=""
byte=sock.recv(1)
while byte!="\r":
chunk+=byte
byte=sock.recv(1)
sock.recv(1)
if chunk.lower()[:9]=="location:":
location=chunk.lower()[9:].strip()
if location=="http://"+servername(url,False)+url: location="/"
locpage=fullpath(location,url)
sock.close()
# if url[len(url)-2:]=="" and locpage[len(locpage)-4:]=="": break
redir=1
if locpage!=url:
redir=0
if pagename(sre.sub("\\\\(\"|\')","\\1",locpage))==pagename(url):
print "QUOTE REDIR"
return
print "OLD:",url
print "NEW:",chunk.lower()
print "REDIR:",locpage
return getpage(locpage,redir=redir,realpage=realpage)
if realpage==1:
sock.close()
return
elif realpage==1:
sock.close()
return url
# get headers, ignoring certain HTTP headers
headers=""
type=0
while 1:
chunk=""
byte=sock.recv(1)
if byte=="\r":
sock.recv(1)
break
while byte!="\r":
chunk+=byte
byte=sock.recv(1)
sock.recv(1)
if chunk.lower()[:11]!="set-cookie:" and chunk.lower()[:5]!="date:" and chunk.lower()[:15]!="content-length:" and chunk.lower()[:11]!="keep-alive:" and chunk.lower()[:18]!="transfer-encoding:" and chunk.lower()[:11]!="connection:":
headers+=chunk
# if chunk.lower()[:15]=="content-length:":
# type=1
# conlen=int(chunk[16:])
if chunk.lower()[:26]=="transfer-encoding: chunked": type=2
# no special type specified, just get the page
if type==0:
body=""
while 1:
chunk=sock.recv(200)
body+=chunk
if chunk=="": break
# set it up if it does have a type
# else:
# byte=sock.recv(1)
# if byte=="\r": sock.recv(1)
# else:
# while 1:
# i=-1
# while byte!="\r":
# i+=1
# byte=sock.recv(1)
# nbytes=sock.recv(3)
# if nbytes=="\n\r\n": break
# # content-length
# if type==1:
# body=""
# for i in range(conlen):
# chunk=sock.recv(1)
# body+=chunk
# transfer-encoding: chunked
if type==2:
body=""
chunksize=""
while chunksize!=0:
byte=""
chunk=""
while byte!="\r":
chunk+=byte
byte=sock.recv(1)
sock.recv(1)
chunksize=int(chunk,16)
wchunksz=chunksize
while wchunksz>=1:
subchunk=sock.recv(wchunksz)
body+=subchunk
wchunksz-=len(subchunk)
sock.recv(2)
# clean up and return
sock.close()
if dheaders!=1: headers=""
return [headers,body,urlfix(url)]
# catch socket errors, such as "connection reset by peer" - trys again until it gives up and goes on to the next page
except socket.error:
print "Socket Error, Recursing..."
return recurse(exceptions)
while 1:
nline = readline()
if nline.strip()[0] != '$':
print "DOOKY", line
if checkerror():
nline = None
break
else:
break
if nline is None:
break
sleep(1)
#if checkerror():
# continue
lineno = sre.sub(r'\$[0-9]+ = ([0-9]+)', r'\1', nline)
#findprompt()
read(5)
send('p ((char*)&((PyStringObject*)f->f_code->co_filename)->ob_sval)\n')
#if checkerror():
# continue
#while 1:
# nline = readline()
# if nline.strip()[0] != '$':
# print "DOOKY2", line
# if checkerror():
# nline = None
# break
# else:
# break
#if nline is None:
#! /usr/bin/python
import os
import sre
import sys
if len(sys.argv) < 1:
raise "pls input some path!"
sys.argv = sys.argv[1:]
print "input arguments is %s" % sys.argv
for i in range(0, len(sys.argv), 1):
print "dir is %s" % sys.argv[i]
if not os.path.isdir(sys.argv[i]):
raise "pls input a valid path!"
flist = os.listdir(sys.argv[i])
print "fileslist is %s" % flist
for f in range(0, len(flist), 1):
print "filename is: %s" % flist[f]
newname = sre.sub(r' +', '_', flist[f])
print newname, flist[f]
print "rename file..."
os.rename(sys.argv[i] + os.sep + flist[f],
sys.argv[i] + os.sep + newname)
def removeurls(body): return sre.sub("http://([^ \"\']+)","",body)
# function to retreive the full path of a url based on the current page url
def fullpath(url,pageurl):
def fullxplt(worldbody,theurl,vars,qry,post,poststring=""):
global pagetype,xssscan,sqliscan,checklevel
sqlivulns=[]
xssvulns=[]
worldbody=decapitate(worldbody)
for i in range(len(vars)):
namval=vars[i]
name=namval[:namval.find("=")]
# if name=="xss": return
value=namval[namval.find("=")+1:]
if pagetype=="php" and name.lower()=="phpsessid":
continue
# SQLI (SQL Injection) Vulnerability Checker
if sqliscan:
worldbody=parsebody(worldbody)
if checklevel<=2:
if sqli_intcheck(worldbody,theurl,qry,namval,post,poststring): sqlivulns.append(name)
else:
bodychk=decapitate(exploit(theurl,qry,namval,"'\"",post,poststring))
if worldbody!=bodychk:
if sqli_check(worldbody,theurl,qry,namval,"",post,poststring): sqlivulns.append(name)
elif sqli_check(worldbody,theurl,qry,namval,"'",post,poststring): sqlivulns.append(name)
elif sqli_check(worldbody,theurl,qry,namval,'"',post,poststring): sqlivulns.append(name)
# XSS (Cross Site Scripting) Vulnerability Checker
if xssscan:
if checklevel<=2:
try:
value=int(value)
xssplt=str(randident_num())
except ValueError:
if checklevel==1: continue
xssplt=randident()
xssplt+="<script>"
if post==0: body=getpage(theurl+sre.sub(namval+"[^\&]*(?=\&|$)",namval+xssplt,qry),poststring=poststring)
else: body=getpage(theurl,poststring=sre.sub(namval+"[^\&]*(?=\&|$)",namval+xssplt,qry))
if not listempty(body):
body=body[1]
if body.count(xssplt)!=0:
xssvulns.append(name)
continue
else:
xssplt=randident()+"<d>"
#xssplt=randident()+randident() # old method
if post==0: body=getpage(theurl+sre.sub(namval+"[^\&]*(?=\&|$)",namval+xssplt,qry),poststring=poststring)
else: body=getpage(theurl,poststring=sre.sub(namval+"[^\&]*(?=\&|$)",namval+xssplt,qry))
if not listempty(body):
body=body[1]
if body.count(xssplt)!=0:
xssvulns.append(name)
continue
xssplt=randident()+'"fg'
if post==0: body=getpage(theurl+sre.sub(namval+"[^\&]*(?=\&|$)",namval+xssplt,qry),poststring=poststring)
else: body=getpage(theurl,poststring=sre.sub(namval+"[^\&]*(?=\&|$)",namval+xssplt,qry))
if not listempty(body):
body=body[1]
if body.count(xssplt)!=0:
xssvulns.append(name)
continue
return [sqlivulns,xssvulns]
def treepages(url, level):
global treeglob, urlfields, postfields, treedurls, levels, server, vulnlogfile, scanlimit, ignorefileext
print ">>>>>>>>", level, "<<<<<<<<"
print " ---> " + url
pageinfo = getpage(url)
if listempty(pageinfo): return
body = pageinfo[1].lower()
print "AA"
# select/option, textarea
# check for forms
bodyarr = sre.split("<form", body)
for i in range(len(bodyarr)):
frmsect = bodyarr[i][:bodyarr[i].find(">")]
frmbody = bodyarr[i][bodyarr[i].find(">"):][:bodyarr[i].find("</form>"
)]
actionurl = getattrval(frmsect, "action")
if actionurl == "" or actionurl == frmsect or actionurl == "\"\"":
actionurl = pageinfo[2]
if actionurl.count(";") > 0:
actionurl = actionurl[actionurl.find(";") + 1:]
if actionurl[:11].lower() == "javascript:": continue
actionurl = fullpath(actionurl, pageinfo[2])
print "ACTION:", actionurl
# get the input variables
poststring = ""
inputarr = sre.sub("(.*?)\<input([^\>]*)\>(.*?)", "\\2|ZZaaXXaaZZ|",
frmbody).split("|ZZaaXXaaZZ|")
for j in range(len(inputarr)):
name = getattrval(inputarr[j], "name")
if name == inputarr[j] or name == "" or name == "\"\"": continue
value = getattrval(inputarr[j], "value")
if value == inputarr[j] or value == "" or value == "\"\"":
value = ""
if poststring != "": poststring += "&"
poststring += name + "=" + value
# get select/option tags
selectarr = sre.sub("(.*?)\<select([^\>]*)\>(.*?)", "\\2|ZZaaXXaaZZ|",
frmbody).split("|ZZaaXXaaZZ|")
for j in range(len(selectarr)):
name = getattrval(selectarr[j], "name")
if name == selectarr[j] or name == "" or name == "\"\"": continue
value = sre.sub(
"(.*?)\<option([^\>]*)value=(\"|'|)([^\\3\ ]*)\\3([^\>]*)\>(.*?)",
"\\2", selectarr[j])
if value == selectarr[j] or value == "" or value == "\"\"":
value = ""
if poststring != "": poststring += "&"
poststring += name + "=" + value
print "sel/opt: " + name + "=" + value
if poststring == "": continue
if sre.search("method=([\'\"]|)post([\'\"]|)",
frmsect[:frmsect.find(">")].lower()) == None:
if actionurl.find("?") != -1: actionurl += "&"
else: actionurl += "?"
actionurl += poststring
body += '<a href="' + actionurl + '">'
print 'GETT <a href="' + actionurl + '">'
continue
# determine if it needs to be scanned, and if so, scan it
postscan = 0
postvars = poststring.split("&")
if postfields.has_key(actionurl):
for j in range(len(postvars)):
postvars[j] = postvars[j][:postvars[j].find("=")]
if postfields[actionurl].count(postvars[j]) == 0:
postfields[actionurl].append(postvars[j])
postscan = 1
else:
for j in range(len(postvars)):
postvars[j] = postvars[j][:postvars[j].find("=")]
postfields[actionurl] = postvars
postscan = 1
if postscan == 1:
vulns = checkvars(actionurl, poststring)
if not listempty(vulns): dispvulns(vulns, actionurl)
print "BB"
# check for urls in "href" tags
# ? # part of 3? (src|href|location|window.open)= and http://
urlreg = "(\'|\")(?!javascript:)(([^\>]+?)(?!\.(" + ignorefileext.replace(
",", "|") + "))(.{3,8}?)(|\?([^\>]+?)))"
urlarr = sre.sub(
"(?s)(?i)(.+?)((src|href)=|location([\ ]*)=([\ ]*)|window\.open\()" +
urlreg + "\\6", "\\7|ZZaaXXaaZZ|", body).split("|ZZaaXXaaZZ|")
del urlarr[len(urlarr) - 1]
urlarr.append(
sre.sub("(?s)(?i)(.+?)(src|href)=" + urlreg + "\\3", "\\4|ZZaaXXaaZZ|",
body).split("|ZZaaXXaaZZ|"))
del urlarr[len(urlarr) - 1]
for i in range(len(urlarr)):
theurl = fullpath(urlarr[i], pageinfo[2])
if not checkserver(servername(theurl)): continue
# determine if it needs scanned and/or treed, and if so, scan and/or tree it
getscan = 0
if theurl.count("?") != 0:
nqurl = theurl[:theurl.find("?")]
query = theurl[theurl.find("?") + 1:]
query = sre.sub("\&\;", "\&", query)
qryvars = query.split("&")
if urlfields.has_key(nqurl):
for j in range(len(qryvars)):
qryvars[j] = qryvars[j][:qryvars[j].find("=")]
if urlfields[nqurl].count(qryvars[j]) == 0:
urlfields[nqurl].append(qryvars[j])
getscan = 1
else:
for j in range(len(qryvars)):
qryvars[j] = qryvars[j][:qryvars[j].find("=")]
urlfields[nqurl] = qryvars
getscan = 1
else:
if urlfields.has_key(theurl) == False: urlfields[theurl] = []
nqurl = theurl
if getscan == 1:
vulns = checkvars(theurl)
if not listempty(vulns): dispvulns(vulns, theurl)
tree = treeglob
if treedurls.has_key(nqurl):
if treedurls[nqurl].count(theurl) == 0 and len(
treedurls[nqurl]) <= scanlimit:
treedurls[nqurl].append(theurl)
else:
tree = 0
else:
treedurls[nqurl] = [theurl]
if tree == 1 and level < levels:
realurl = getpage(theurl, realpage=1)
if theurl != realurl and realurl != None:
body += ' href="' + realurl + '" '
print "treeee"
try:
treepages(theurl, level + 1)
except KeyboardInterrupt:
treeglob = 0
print "TREEGLOB CHANGED TO ZERO"
treepages(theurl, level + 1)
def fullxplt(worldbody, theurl, vars, qry, post, poststring=""):
global pagetype, xssscan, sqliscan, checklevel
sqlivulns = []
xssvulns = []
worldbody = decapitate(worldbody)
for i in range(len(vars)):
namval = vars[i]
name = namval[:namval.find("=")]
# if name=="xss": return
value = namval[namval.find("=") + 1:]
if pagetype == "php" and name.lower() == "phpsessid":
continue
# SQLI (SQL Injection) Vulnerability Checker
if sqliscan:
worldbody = parsebody(worldbody)
if checklevel <= 2:
if sqli_intcheck(worldbody, theurl, qry, namval, post,
poststring):
sqlivulns.append(name)
else:
bodychk = decapitate(
exploit(theurl, qry, namval, "'\"", post, poststring))
if worldbody != bodychk:
if sqli_check(worldbody, theurl, qry, namval, "", post,
poststring):
sqlivulns.append(name)
elif sqli_check(worldbody, theurl, qry, namval, "'", post,
poststring):
sqlivulns.append(name)
elif sqli_check(worldbody, theurl, qry, namval, '"', post,
poststring):
sqlivulns.append(name)
# XSS (Cross Site Scripting) Vulnerability Checker
if xssscan:
if checklevel <= 2:
try:
value = int(value)
xssplt = str(randident_num())
except ValueError:
if checklevel == 1: continue
xssplt = randident()
xssplt += "<script>"
if post == 0:
body = getpage(theurl + sre.sub(namval + "[^\&]*(?=\&|$)",
namval + xssplt, qry),
poststring=poststring)
else:
body = getpage(theurl,
poststring=sre.sub(
namval + "[^\&]*(?=\&|$)",
namval + xssplt, qry))
if not listempty(body):
body = body[1]
if body.count(xssplt) != 0:
xssvulns.append(name)
continue
else:
xssplt = randident() + "<d>"
#xssplt=randident()+randident() # old method
if post == 0:
body = getpage(theurl + sre.sub(namval + "[^\&]*(?=\&|$)",
namval + xssplt, qry),
poststring=poststring)
else:
body = getpage(theurl,
poststring=sre.sub(
namval + "[^\&]*(?=\&|$)",
namval + xssplt, qry))
if not listempty(body):
body = body[1]
if body.count(xssplt) != 0:
xssvulns.append(name)
continue
xssplt = randident() + '"fg'
if post == 0:
body = getpage(theurl + sre.sub(namval + "[^\&]*(?=\&|$)",
namval + xssplt, qry),
poststring=poststring)
else:
body = getpage(theurl,
poststring=sre.sub(
namval + "[^\&]*(?=\&|$)",
namval + xssplt, qry))
if not listempty(body):
body = body[1]
if body.count(xssplt) != 0:
xssvulns.append(name)
continue
return [sqlivulns, xssvulns]
def rename_page(env, oldname, newname, user, ip, debug=False, db=None):
"""Rename a wiki page from oldname to newname, using env as the environment."""
handle_commit = False
if not db:
db = env.get_db_cnx()
handle_commit = True
cursor = db.cursor()
if debug is False:
debug = pass_
elif debug is True:
debug = print_
else:
debug = log_intercept(debug)
sqlbase = ' FROM wiki w1, ' + \
'(SELECT name, MAX(version) AS max_version FROM wiki GROUP BY name) w2 ' + \
'WHERE w1.version = w2.max_version AND w1.name = w2.name '
sql = 'SELECT w1.version,w1.text' + sqlbase + 'AND w1.name = \'%s\'' % oldname
debug('Running query %r', sql)
cursor.execute(sql)
row = cursor.fetchone()
if not row:
raise TracError, 'Page not found'
new_wiki_page = (newname, row[0] + 1, int(time.time()), user, ip, row[1],
'Name changed from %s to %s' % (oldname, newname), 0)
# Create a new page with the needed comment
debug('Inserting new page %r', new_wiki_page)
cursor.execute(
'INSERT INTO wiki (name,version,time,author,ipnr,text,comment,readonly) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)',
new_wiki_page)
# Move all the old versions of the page
debug("Moving all old versions of page")
cursor.execute('UPDATE wiki SET name=%s WHERE name=%s', (newname, oldname))
# Move any attachments that are on the page
debug("Moving all attachments in database")
cursor.execute('UPDATE attachment SET id=%s WHERE type=%s AND id=%s',
(newname, 'wiki', oldname))
debug("Found %s attachments on that page", cursor.rowcount)
if cursor.rowcount > 0:
# Change the directory where the attachments are stored, if there were any
debug('Moving all attachments on file system')
from_path = os.path.join(env.path, 'attachments', 'wiki',
urllib.quote(oldname))
to_path = os.path.join(env.path, 'attachments', 'wiki',
urllib.quote(newname))
debug('Moving from %r to %r', from_path, to_path)
os.renames(from_path, to_path)
# Get a list of all wiki pages containing links to the old page
debug("Trying to fix links")
sql = 'SELECT w1.version,w1.name,w1.text' + sqlbase + "AND w1.text like '%%[wiki:%s%%'" % oldname
debug('Running query %r', sql)
cursor.execute(sql)
# Rewrite all links to the old page, such as to point to the new page
for row in list(cursor):
debug("Found a page with a backlink in it: %s (v%s)", row[1], row[0])
newtext = sre.sub('\[wiki:%s' % oldname, '[wiki:%s' % newname, row[2])
cursor.execute('UPDATE wiki SET text=%s WHERE name=%s AND version=%s',
(newtext, row[1], row[0]))
if handle_commit:
db.commit()
def treepages(url,level):
global treeglob,urlfields,postfields,treedurls,levels,server,vulnlogfile,scanlimit,ignorefileext
print ">>>>>>>>",level,"<<<<<<<<"
print " ---> "+url
pageinfo=getpage(url)
if listempty(pageinfo): return
body=pageinfo[1].lower()
print "AA"
# select/option, textarea
# check for forms
bodyarr=sre.split("<form",body)
for i in range(len(bodyarr)):
frmsect=bodyarr[i][:bodyarr[i].find(">")]
frmbody=bodyarr[i][bodyarr[i].find(">"):][:bodyarr[i].find("</form>")]
actionurl=getattrval(frmsect,"action")
if actionurl=="" or actionurl==frmsect or actionurl=="\"\"": actionurl=pageinfo[2]
if actionurl.count(";")>0: actionurl=actionurl[actionurl.find(";")+1:]
if actionurl[:11].lower()=="javascript:": continue
actionurl=fullpath(actionurl,pageinfo[2])
print "ACTION:",actionurl
# get the input variables
poststring=""
inputarr=sre.sub("(.*?)\<input([^\>]*)\>(.*?)","\\2|ZZaaXXaaZZ|",frmbody).split("|ZZaaXXaaZZ|")
for j in range(len(inputarr)):
name=getattrval(inputarr[j],"name")
if name==inputarr[j] or name=="" or name=="\"\"": continue
value=getattrval(inputarr[j],"value")
if value==inputarr[j] or value=="" or value=="\"\"": value=""
if poststring!="": poststring+="&"
poststring+=name+"="+value
# get select/option tags
selectarr=sre.sub("(.*?)\<select([^\>]*)\>(.*?)","\\2|ZZaaXXaaZZ|",frmbody).split("|ZZaaXXaaZZ|")
for j in range(len(selectarr)):
name=getattrval(selectarr[j],"name")
if name==selectarr[j] or name=="" or name=="\"\"": continue
value=sre.sub("(.*?)\<option([^\>]*)value=(\"|'|)([^\\3\ ]*)\\3([^\>]*)\>(.*?)","\\2",selectarr[j])
if value==selectarr[j] or value=="" or value=="\"\"": value=""
if poststring!="": poststring+="&"
poststring+=name+"="+value
print "sel/opt: "+name+"="+value
if poststring=="": continue
if sre.search("method=([\'\"]|)post([\'\"]|)",frmsect[:frmsect.find(">")].lower())==None:
if actionurl.find("?")!=-1: actionurl+="&"
else: actionurl+="?"
actionurl+=poststring
body+='<a href="'+actionurl+'">'
print 'GETT <a href="'+actionurl+'">'
continue
# determine if it needs to be scanned, and if so, scan it
postscan=0
postvars=poststring.split("&")
if postfields.has_key(actionurl):
for j in range(len(postvars)):
postvars[j]=postvars[j][:postvars[j].find("=")]
if postfields[actionurl].count(postvars[j])==0:
postfields[actionurl].append(postvars[j])
postscan=1
else:
for j in range(len(postvars)): postvars[j]=postvars[j][:postvars[j].find("=")]
postfields[actionurl]=postvars
postscan=1
if postscan==1:
vulns=checkvars(actionurl,poststring)
if not listempty(vulns): dispvulns(vulns,actionurl)
print "BB"
# check for urls in "href" tags
# ? # part of 3? (src|href|location|window.open)= and http://
urlreg="(\'|\")(?!javascript:)(([^\>]+?)(?!\.("+ignorefileext.replace(",","|")+"))(.{3,8}?)(|\?([^\>]+?)))"
urlarr=sre.sub("(?s)(?i)(.+?)((src|href)=|location([\ ]*)=([\ ]*)|window\.open\()"+urlreg+"\\6","\\7|ZZaaXXaaZZ|",body).split("|ZZaaXXaaZZ|")
del urlarr[len(urlarr)-1]
urlarr.append(sre.sub("(?s)(?i)(.+?)(src|href)="+urlreg+"\\3","\\4|ZZaaXXaaZZ|",body).split("|ZZaaXXaaZZ|"))
del urlarr[len(urlarr)-1]
for i in range(len(urlarr)):
theurl=fullpath(urlarr[i],pageinfo[2])
if not checkserver(servername(theurl)): continue
# determine if it needs scanned and/or treed, and if so, scan and/or tree it
getscan=0
if theurl.count("?")!=0:
nqurl=theurl[:theurl.find("?")]
query=theurl[theurl.find("?")+1:]
query=sre.sub("\&\;","\&",query)
qryvars=query.split("&")
if urlfields.has_key(nqurl):
for j in range(len(qryvars)):
qryvars[j]=qryvars[j][:qryvars[j].find("=")]
if urlfields[nqurl].count(qryvars[j])==0:
urlfields[nqurl].append(qryvars[j])
getscan=1
else:
for j in range(len(qryvars)): qryvars[j]=qryvars[j][:qryvars[j].find("=")]
urlfields[nqurl]=qryvars
getscan=1
else:
if urlfields.has_key(theurl)==False: urlfields[theurl]=[]
nqurl=theurl
if getscan==1:
vulns=checkvars(theurl)
if not listempty(vulns): dispvulns(vulns,theurl)
tree=treeglob
if treedurls.has_key(nqurl):
if treedurls[nqurl].count(theurl)==0 and len(treedurls[nqurl])<=scanlimit:
treedurls[nqurl].append(theurl)
else: tree=0
else: treedurls[nqurl]=[theurl]
if tree==1 and level<levels:
realurl=getpage(theurl,realpage=1)
if theurl!=realurl and realurl!=None:
body+=' href="'+realurl+'" '
print "treeee"
try: treepages(theurl,level+1)
except KeyboardInterrupt:
treeglob=0
print "TREEGLOB CHANGED TO ZERO"
treepages(theurl,level+1)
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------
#
# parsetestResults.pl
#
# Transliteration of the perl version
#
import sys,sre
docStrings = {}
for line in sys.stdin:
if not sre.match("^-",line):
x=sre.match("^([^-]+)-(.*)... ok$",line)
if x:
method,description = x.groups()
try:
docStrings[method].append(description)
except KeyError:
docStrings[method]=[description]
methods = [method for method in docStrings]
methods.sort()
for method in methods:
methPrint = sre.sub("(^ *| *$)", "", method)
print "\t$ ==%s== : " % methPrint,;
descriptions = docStrings[method]
descriptions.sort()
for description in descriptions:
print "\t-%s<br>" % description
def removeurls(body):
return sre.sub("http://([^ \"\']+)", "", body)
import gzip
import sre
import binascii
d1 = open("d1.txt")
aout = open("18a.out", "w")
for line in d1.readlines():
print sre.sub("[ \n]", "", line)
aout.write(binascii.unhexlify(sre.sub("[ \n]", "", line)))
d2 = open("d2.txt")
bout = open("18b.out", "w")
for line in d2.readlines():
bout.write(binascii.unhexlify(sre.sub("[ \n]", "", line)))
def scanSGF (target):
if isdir(target):
for sgf_file in os.listdir(target):
if sgf_file!='Desktop':
scanSGF(target+os.sep+sgf_file)
elif isfile(target) and target[len(target)-4:]=='.sgf':
try:
#Check to see if there is already a result tag. At the moment, it will accept any sort
#of result tag, but this is easily modified to replace result tags that are improper.
if len(sre.findall('(RE\[.*\])',file(target,'r').read())) >0:
print target+" already has a result. Skipping..."
else:
print target+":",
next_move = sre.findall('([B|W])\[[a-z][a-z]\]',file(target,'r').read())[-1]
#next_move looks inside the SGF to find the last player who made a move. This is so that later, the
#GnuGo engine knows for which player to generate a move.
if next_move=='W':
next_move='black'
else:
next_move='white'
#The black/white syntax is needed by the GTP protocol.
gtp_test = file('gtp_test','w')
gtp_test.write('reg_genmove '+next_move+'\ntop_moves\nquit')
gtp_test.flush()
#Although it would technically be possible to bind gnugo's STDIN and STDOUT to Python, it is just
#so much simpler to put the commands in a file. The file is deleted later anyway.
gnugo_session = os.popen('gnugo --mode gtp --gtp-input gtp_test -l '+target).read()
if len(sre.findall('PASS',gnugo_session))>0:
move_value = 0
#If GnuGo thinks the best move is to pass, then the game is obviously over, and setting
#move_value to 0 will ensure that the game will later be given to GnuGo to estimate score.
else:
move_value = sre.findall('([\d\d|\d]\.[\d|\d\d])',gnugo_session)[0]
#Since GnuGo will give the values of the move in reverse order that they are played, the
#value of the most recent move (which we generated in gtp_test) will be the first one.
#This is the value we want to check for size.
if next_move=='black':
next_move='W'
else:
next_move='B'
#I am making an assumption here, that the last person to move is going to win the game.
#It seems silly for a player to make a move and then resign, but it is not an impossibility.
#Therefore, if you need that extra bit of accuracy, you can make some minor modifications
#to check the estimated score regardless of whether the game ended in resign or not, and
#use that as a sign of who won.
game_result = next_move+'+R'
if float(move_value)<2:
#If the value of the move generated by GnuGo is less than 2, then it is clear that the late
#endgame has been reached, and the game is probably over. In this case, we will use GnuGo
#to calculate the relative score.
result_string = os.popen('gnugo -l '+target+' --score estimate').read()
winning_color = result_string[:1]
score_estimate = sre.findall('(\d.\d)',result_string)[0]
game_result = winning_color+'+'+score_estimate
print game_result
sgf_raw = file(target,'r')
file_dump = sgf_raw.read()
file_dump = sre.sub(RE_PREFIX,PREFIX+'RE['+game_result+']',file_dump)
sgf_write=file(target,'w')
sgf_write.write(file_dump)
sgf_write.flush()
sgf_write.close()
os.remove('gtp_test')
#Remove the old gtp_test.
except IndexError:
print "Error with SGF "+target+". Deleting ..."
error_log = file('error_log','a')
error_log.write("Error on "+target+". Deleting file.\n")
error_log.flush()
error_log.close()
os.remove(target)
#Uncomment previous line if you would like to keep illegal SGF's.
except Exception:
print "Error. Skipping ..."
print Exception
error_log = file('error_log','a')
error_log.write("Error on "+target+". Skipping file.\n")
error_log.flush()
error_log.close()
except Error:
print "Error. Skipping ..."
print Error
error_log = file('error_log','a')
error_log.write("Error on "+target+". Skipping file.\n")
error_log.flush()
error_log.close()
import zipfile
import sre
comments = ""
z = zipfile.ZipFile(open("channel.zip"))
text = z.read("90052.txt")
while text:
print text
text = sre.sub("Next nothing is ", "", text)
try:
comments += z.getinfo(text + ".txt").comment
text = z.read(text + ".txt")
except KeyError:
print comments
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------
#
# parsetestResults.pl
#
# Transliteration of the perl version
#
import sys, sre
docStrings = {}
for line in sys.stdin:
if not sre.match("^-", line):
x = sre.match("^([^-]+)-(.*)... ok$", line)
if x:
method, description = x.groups()
try:
docStrings[method].append(description)
except KeyError:
docStrings[method] = [description]
methods = [method for method in docStrings]
methods.sort()
for method in methods:
methPrint = sre.sub("(^ *| *$)", "", method)
print "\t$ ==%s== : " % methPrint,
descriptions = docStrings[method]
descriptions.sort()
for description in descriptions:
print "\t-%s<br>" % description
def __str__(self):
ret = super(LimbedGameObj, self).__str__().replace(" ", "L", 1) # To make it line up with " GO" lines
ret += "\n" + sre.sub(self._tabberpat, " ", str(self.limbs)).rstrip()
return ret
import gzip
import sre
import binascii
import difflib
f = gzip.GzipFile("deltas.gz")
left = []
right = []
for line in f.readlines():
(a,b) = sre.match("^(.{53}) (.*)", line).groups()
left.append(a.strip() + "\n")
right.append(b.strip() + "\n")
streams = {}
for line in difflib.ndiff(left,right):
key = line[0]
if not streams.has_key(key):
streams[key] = open("level18%s.out"%key, "w")
data = binascii.unhexlify(sre.sub("[ \n]","",line[2:]))
if len(data) > 0:
streams[key].write(data)
for key in streams.keys():
print streams[key].name
streams[key].close()